From: Liam R. Howlett Date: Mon, 14 Dec 2020 18:14:06 +0000 (-0500) Subject: maple_tree: Stop using linked list in most cases X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=bb2f53af9c9b4e049922e4795f4aedc3df83ac9c;p=users%2Fjedix%2Flinux-maple.git maple_tree: Stop using linked list in most cases Signed-off-by: Liam R. Howlett --- diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index debb8995d57f..caa7d8f6b99d 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -144,10 +144,12 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + rcu_read_lock(); + mas_for_each(&mas, vma, ULONG_MAX) { unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) @@ -157,6 +159,7 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) zap_page_range(vma, vma->vm_start, size); #endif } + rcu_read_unlock(); mmap_read_unlock(mm); return 0; diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 86a1a63563fd..288a25e1b1c2 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -520,8 +520,9 @@ static inline unsigned long mm_total_size(struct mm_struct *mm) { struct vm_area_struct *vma; unsigned long usize = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) + mas_for_each(&mas, vma, ULONG_MAX) usize += vma->vm_end - vma->vm_start; return usize; } @@ -548,6 +549,7 @@ void flush_cache_mm(struct mm_struct *mm) { struct vm_area_struct *vma; pgd_t *pgd; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* Flushing the whole cache on each cpu takes forever on rp3440, etc. So, avoid it if the mm isn't too big. */ @@ -560,7 +562,7 @@ void flush_cache_mm(struct mm_struct *mm) } if (mm->context == mfsp(3)) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { flush_user_dcache_range_asm(vma->vm_start, vma->vm_end); if (vma->vm_flags & VM_EXEC) flush_user_icache_range_asm(vma->vm_start, vma->vm_end); @@ -570,7 +572,7 @@ void flush_cache_mm(struct mm_struct *mm) } pgd = mm->pgd; - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { unsigned long addr; for (addr = vma->vm_start; addr < vma->vm_end; diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c index b6c7427daa6f..be595b36dc4c 100644 --- a/arch/powerpc/mm/book3s32/tlb.c +++ b/arch/powerpc/mm/book3s32/tlb.c @@ -121,6 +121,7 @@ EXPORT_SYMBOL(flush_tlb_kernel_range); void flush_tlb_mm(struct mm_struct *mm) { struct vm_area_struct *mp; + MA_STATE(mas, &mm->mm_mt, 0, 0); if (!Hash) { _tlbia(); @@ -133,7 +134,7 @@ void flush_tlb_mm(struct mm_struct *mm) * unmap_region or exit_mmap, but not from vmtruncate on SMP - * but it seems dup_mmap is the only SMP case which gets here. */ - for (mp = mm->mmap; mp != NULL; mp = mp->vm_next) + mas_for_each(&mas, mp, ULONG_MAX) flush_range(mp->vm_mm, mp->vm_start, mp->vm_end); } EXPORT_SYMBOL(flush_tlb_mm); diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c index 60c6ea16a972..ada1531f81be 100644 --- a/arch/powerpc/mm/book3s64/subpage_prot.c +++ b/arch/powerpc/mm/book3s64/subpage_prot.c @@ -149,24 +149,15 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, unsigned long len) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, addr, addr); /* * We don't try too hard, we just mark all the vma in that range * VM_NOHUGEPAGE and split them. */ - vma = find_vma(mm, addr); - /* - * If the range is in unmapped range, just return - */ - if (vma && ((addr + len) <= vma->vm_start)) - return; - - while (vma) { - if (vma->vm_start >= (addr + len)) - break; + mas_for_each(&mas, vma, addr + len) { vma->vm_flags |= VM_NOHUGEPAGE; walk_page_vma(vma, &subpage_walk_ops, NULL); - vma = vma->vm_next; } } #else diff --git a/arch/powerpc/oprofile/cell/spu_task_sync.c b/arch/powerpc/oprofile/cell/spu_task_sync.c index 489f993100d5..c1d59a255060 100644 --- a/arch/powerpc/oprofile/cell/spu_task_sync.c +++ b/arch/powerpc/oprofile/cell/spu_task_sync.c @@ -321,6 +321,7 @@ get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, struct vm_area_struct *vma; struct file *exe_file; struct mm_struct *mm = spu->mm; + MA_STATE(mas, &mm->mm_mt, spu_ref, spu_ref); if (!mm) goto out; @@ -333,19 +334,17 @@ get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, } mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) - continue; - my_offset = spu_ref - vma->vm_start; - if (!vma->vm_file) - goto fail_no_image_cookie; - - pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n", - my_offset, spu_ref, vma->vm_file); - *offsetp = my_offset; - break; - } + vma = mas_walk(&mas); + if (!vma) + goto fail_no_image_cookie; + + my_offset = spu_ref - vma->vm_start; + if (!vma->vm_file) + goto fail_no_image_cookie; + pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n", + my_offset, spu_ref, vma->vm_file); + *offsetp = my_offset; *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path); pr_debug("got dcookie for %pD\n", vma->vm_file); diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 64795d034926..8f7ed79583b2 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2502,8 +2502,9 @@ static const struct mm_walk_ops thp_split_walk_ops = { static inline void thp_split_mm(struct mm_struct *mm) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { vma->vm_flags &= ~VM_HUGEPAGE; vma->vm_flags |= VM_NOHUGEPAGE; walk_page_vma(vma, &thp_split_walk_ops, NULL); @@ -2571,8 +2572,9 @@ int gmap_mark_unmergeable(void) struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int ret; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, MADV_UNMERGEABLE, &vma->vm_flags); if (ret) diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index 61776790cd67..e40dd6deb1d2 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -590,21 +590,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm(struct mm_struct *mm) { - struct vm_area_struct *vma = mm->mmap; + struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - while (vma != NULL) { + mas_for_each(&mas, vma, ULONG_MAX) fix_range(mm, vma->vm_start, vma->vm_end, 0); - vma = vma->vm_next; - } } void force_flush_all(void) { struct mm_struct *mm = current->mm; - struct vm_area_struct *vma = mm->mmap; + struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - while (vma != NULL) { + mas_for_each(&mas, vma, ULONG_MAX) fix_range(mm, vma->vm_start, vma->vm_end, 1); - vma = vma->vm_next; - } } diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 9185cb1d13b9..f531efb00ba3 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -144,9 +144,11 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) struct mm_struct *mm = task->mm; struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); + mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, &vvar_mapping)) @@ -371,6 +373,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_write_lock(mm); /* @@ -380,7 +383,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) * We could search vma near context.vdso, but it's a slowpath, * so let's explicitly check all VMAs to be completely sure. */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (vma_is_special_mapping(vma, &vdso_mapping) || vma_is_special_mapping(vma, &vvar_mapping)) { mmap_write_unlock(mm); diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c index 2c415fce6801..26ec2e67879a 100644 --- a/arch/xtensa/kernel/syscall.c +++ b/arch/xtensa/kernel/syscall.c @@ -62,6 +62,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct vm_area_struct *vmm; + MA_STATE(mas, &mm->mm_mt, addr, addr); if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -83,7 +84,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, else addr = PAGE_ALIGN(addr); - for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { + mas_for_each(&mas, vmm, ULONG_MAX) { /* At this point: (!vmm || addr < vmm->vm_end). */ if (TASK_SIZE - len < addr) return -ENOMEM; diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index 01153b74334a..47951b84f2cd 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -313,6 +313,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx) struct vm_area_struct *vma; int rc; struct mm_struct *mm; + MA_STATE(mas, &mm->mm_mt, 0, 0); mm = get_mem_context(ctx); if (mm == NULL) { @@ -322,7 +323,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx) } mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { for (ea = vma->vm_start; ea < vma->vm_end; ea = next_segment(ea, slb.vsid)) { rc = copro_calculate_slb(mm, ea, &slb); diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c index cc917865f13a..fbb43edad41b 100644 --- a/drivers/oprofile/buffer_sync.c +++ b/drivers/oprofile/buffer_sync.c @@ -257,11 +257,8 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) struct vm_area_struct *vma; mmap_read_lock(mm); - for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { - - if (addr < vma->vm_start || addr >= vma->vm_end) - continue; - + vma = find_vma_intersection(mm, addr, addr + 1); + if (vma) { if (vma->vm_file) { cookie = fast_get_dcookie(&vma->vm_file->f_path); *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - @@ -270,12 +267,9 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) /* must be an anonymous map */ *offset = addr; } - - break; - } - - if (!vma) + } else cookie = INVALID_COOKIE; + mmap_read_unlock(mm); return cookie; diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index c981757ba0d4..94acf379eaee 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -545,12 +545,17 @@ static bool is_normal_memory(pgprot_t p) static int __check_mem_type(struct vm_area_struct *vma, unsigned long end) { - while (vma && is_normal_memory(vma->vm_page_prot)) { - if (vma->vm_end >= end) - return 0; - vma = vma->vm_next; + MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_start, vma->vm_start); + + + mas_for_each(&mas, vma, end) { + if (!is_normal_memory(vma->vm_page_prot)) + break; } + if (!vma) + return 0; + return -EINVAL; } diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index fa50e8936f5f..92b67b914bf9 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1609,6 +1609,7 @@ static int fill_files_note(struct memelfnote *note) user_long_t *data; user_long_t *start_end_ofs; char *name_base, *name_curpos; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* *Estimated* file count and total data size needed */ count = mm->map_count; @@ -1633,7 +1634,7 @@ static int fill_files_note(struct memelfnote *note) name_base = name_curpos = ((char *)data) + names_ofs; remaining = size - names_ofs; count = 0; - for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { struct file *file; const char *filename; diff --git a/fs/coredump.c b/fs/coredump.c index c6acfc694f65..0699ce6f6cc1 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -1059,7 +1059,7 @@ static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, { struct vm_area_struct *ret; - ret = this_vma->vm_next; + ret = vma_next(this_vma->vm_mm, this_vma); if (ret) return ret; if (this_vma == gate_vma) diff --git a/fs/exec.c b/fs/exec.c index aa466921d6a9..6eb585ea1b84 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -712,7 +712,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) * when the old and new regions overlap clear from new_end. */ free_pgd_range(&tlb, new_end, old_end, new_end, - vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); + vma_next(mm, vma) ? vma_next(mm, vma)->vm_start : USER_PGTABLES_CEILING); } else { /* * otherwise, clean from old_start; this is done to not touch @@ -721,7 +721,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) * for the others its just a little faster. */ free_pgd_range(&tlb, old_start, old_end, new_end, - vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); + vma_next(mm, vma) ? vma_next(mm, vma)->vm_start : USER_PGTABLES_CEILING); } tlb_finish_mmu(&tlb, old_start, old_end); diff --git a/fs/proc/base.c b/fs/proc/base.c index b362523a9829..0a7d4d3d6ae9 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2316,6 +2316,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) GENRADIX(struct map_files_info) fa; struct map_files_info *p; int ret; + MA_STATE(mas, NULL, 0, 0); genradix_init(&fa); @@ -2343,6 +2344,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) } nr_files = 0; + mas.tree = &mm->mm_mt; /* * We need two passes here: @@ -2354,7 +2356,8 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) * routine might require mmap_lock taken in might_fault(). */ - for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { + pos = 2; + mas_for_each(&mas, vma, ULONG_MAX) { if (!vma->vm_file) continue; if (++pos <= ctx->pos) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 1ec2dd34ebff..20a0fb414c4f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -164,14 +164,13 @@ static void *m_start(struct seq_file *m, loff_t *ppos) static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { struct proc_maps_private *priv = m->private; - struct vm_area_struct *next, *vma = v; + struct vm_area_struct *next = NULL, *vma = v; - if (vma == priv->tail_vma) - next = NULL; - else if (vma->vm_next) - next = vma->vm_next; - else - next = priv->tail_vma; + if (vma != priv->tail_vma) { + next = vma_next(vma->vm_mm, vma); + if (!next) + next = priv->tail_vma; + } *ppos = next ? next->vm_start : -1UL; @@ -930,7 +929,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) smap_gather_stats(vma, &mss, last_vma_end); } /* Case 2 above */ - vma = vma->vm_next; + vma = vma_next(mm, vma); } show_vma_header_prefix(m, priv->mm->mmap->vm_start, @@ -1209,6 +1208,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, return -ESRCH; mm = get_task_mm(task); if (mm) { + MA_STATE(mas, &mm->mm_mt, 0, 0); struct mmu_notifier_range range; struct clear_refs_private cp = { .type = type, @@ -1235,7 +1235,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, } tlb_gather_mmu(&tlb, mm, 0, -1); if (type == CLEAR_REFS_SOFT_DIRTY) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!(vma->vm_flags & VM_SOFTDIRTY)) continue; mmap_read_unlock(mm); @@ -1243,7 +1243,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, count = -EINTR; goto out_mm; } - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_reset(&mas); + mas_set(&mas, 0); + mas_for_each(&mas, vma, ULONG_MAX) { vma->vm_flags &= ~VM_SOFTDIRTY; vma_set_page_prot(vma); } diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 000b457ad087..060e35f4bde9 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -598,14 +598,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, if (release_new_ctx) { struct vm_area_struct *vma; struct mm_struct *mm = release_new_ctx->mm; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* the various vma->vm_userfaultfd_ctx still points to it */ mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) + mas_for_each(&mas, vma, ULONG_MAX) { if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING); } + } mmap_write_unlock(mm); userfaultfd_ctx_put(release_new_ctx); @@ -790,7 +792,9 @@ int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct list_head *unmaps) { - for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { + MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); + + mas_for_each(&mas, vma, end) { struct userfaultfd_unmap_ctx *unmap_ctx; struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; @@ -840,6 +844,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) /* len == 0 means wake all */ struct userfaultfd_wake_range range = { .len = 0, }; unsigned long new_flags; + MA_STATE(mas, &mm->mm_mt, 0, 0); WRITE_ONCE(ctx->released, true); @@ -856,7 +861,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) */ mmap_write_lock(mm); prev = NULL; - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { cond_resched(); BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); @@ -1270,6 +1275,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, bool found; bool basic_ioctls; unsigned long start, end, vma_end; + MA_STATE(mas, &mm->mm_mt, 0, 0); user_uffdio_register = (struct uffdio_register __user *) arg; @@ -1328,7 +1334,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, */ found = false; basic_ioctls = false; - for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { + mas_set(&mas, vma->vm_start); + mas_for_each(&mas, curr, end) { cond_resched(); BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ @@ -1444,7 +1451,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, skip: prev = vma; start = vma->vm_end; - vma = vma->vm_next; + vma = vma_next(vma); } while (vma && vma->vm_start < end); out_unlock: mmap_write_unlock(mm); @@ -1485,6 +1492,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, bool found; unsigned long start, end, vma_end; const void __user *buf = (void __user *)arg; + MA_STATE(mas, &mm->mm_mt, 0, 0); ret = -EFAULT; if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) @@ -1528,7 +1536,9 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, */ found = false; ret = -EINVAL; - for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { + mas_set(&mas, vma->vm_start); + + mas_for_each(&mas, cur, end) { cond_resched(); BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ @@ -1614,7 +1624,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, skip: prev = vma; start = vma->vm_end; - vma = vma->vm_next; + vma = vma_next(vma); } while (vma && vma->vm_start < end); out_unlock: mmap_write_unlock(mm); diff --git a/include/linux/mm.h b/include/linux/mm.h index f039fefa3acd..6a5e4fcbadec 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2654,6 +2654,24 @@ extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned lon extern struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, unsigned long start_addr, unsigned long end_addr); +static inline struct vm_area_struct *vma_next(struct mm_struct *mm, + struct vm_area_struct *vma) +{ + MA_STATE(mas, &mm->mm_mt, 0, 0); + + mas_set(&mas, vma->vm_end); + return mas_next(&mas, ULONG_MAX); +} + +static inline struct vm_area_struct *vma_prev(struct mm_struct *mm, + struct vm_area_struct *vma) +{ + MA_STATE(mas, &mm->mm_mt, 0, 0); + + mas_set(&mas, vma->vm_start); + return mas_prev(&mas, 0); +} + static inline unsigned long vm_start_gap(struct vm_area_struct *vma) { unsigned long vm_start = vma->vm_start; @@ -2695,6 +2713,21 @@ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, return vma; } +static inline struct vm_area_struct *vma_mas_next(struct ma_state *mas) +{ + struct ma_state tmp; + + memcpy(&tmp, mas, sizeof(tmp)); + return mas_next(&tmp, ULONG_MAX); +} + +static inline struct vm_area_struct *vma_mas_prev(struct ma_state *mas) +{ + struct ma_state tmp; + + memcpy(&tmp, mas, sizeof(tmp)); + return mas_prev(&tmp, 0); +} static inline bool range_in_vma(struct vm_area_struct *vma, unsigned long start, unsigned long end) { diff --git a/ipc/shm.c b/ipc/shm.c index e25c7c6106bc..02f4c9413690 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1631,6 +1631,7 @@ long ksys_shmdt(char __user *shmaddr) loff_t size = 0; struct file *file; struct vm_area_struct *next; + MA_STATE(mas, &mm->mm_mt, addr, addr); #endif if (addr & ~PAGE_MASK) @@ -1660,11 +1661,11 @@ long ksys_shmdt(char __user *shmaddr) * match the usual checks anyway. So assume all vma's are * above the starting address given. */ - vma = find_vma(mm, addr); #ifdef CONFIG_MMU + vma = mas_find(&mas, ULONG_MAX); while (vma) { - next = vma->vm_next; + next = mas_find(&mas, ULONG_MAX); /* * Check if the starting address would match, i.e. it's @@ -1702,22 +1703,21 @@ long ksys_shmdt(char __user *shmaddr) * prevent overflows and make comparisons vs. equal-width types. */ size = PAGE_ALIGN(size); - while (vma && (loff_t)(vma->vm_end - addr) <= size) { - next = vma->vm_next; + mas_for_each(&mas, vma, size) { /* finding a matching vma now does not alter retval */ if ((vma->vm_ops == &shm_vm_ops) && ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && (vma->vm_file == file)) do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); - vma = next; } #else /* CONFIG_MMU */ + vma = mas_walk(&mas); /* under NOMMU conditions, the exact address to be destroyed must be * given */ - if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) { + if (vma && vma->vm_ops == &shm_vm_ops) { do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); retval = 0; } diff --git a/kernel/acct.c b/kernel/acct.c index f175df8f6aa4..c1cb3d68948e 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -545,7 +545,7 @@ void acct_collect(long exitcode, int group_dead) vma = current->mm->mmap; while (vma) { vsize += vma->vm_end - vma->vm_start; - vma = vma->vm_next; + vma = vma_next(vma->vm_mm, vma); } mmap_read_unlock(current->mm); } diff --git a/kernel/events/core.c b/kernel/events/core.c index dc568ca295bd..767bec30c395 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9888,8 +9888,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, struct perf_addr_filter_range *fr) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!vma->vm_file) continue; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 00b0358739ab..b0aa05be2a61 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -356,8 +356,9 @@ static struct vm_area_struct * find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) { struct vm_area_struct *tmp; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) + mas_for_each(&mas, tmp, ULONG_MAX) if (valid_ref_ctr_vma(uprobe, tmp)) return tmp; @@ -1239,9 +1240,10 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) { struct vm_area_struct *vma; int err = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { unsigned long vaddr; loff_t offset; @@ -1990,8 +1992,9 @@ bool uprobe_deny_signal(void) static void mmf_recalc_uprobes(struct mm_struct *mm) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!valid_vma(vma, false)) continue; /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ae7ceba8fd4f..710727741538 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2723,6 +2723,7 @@ static void task_numa_work(struct callback_head *work) unsigned long start, end; unsigned long nr_pte_updates = 0; long pages, virtpages; + MA_STATE(mas, &mm->mm_mt, 0, 0); SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); @@ -2775,13 +2776,17 @@ static void task_numa_work(struct callback_head *work) if (!mmap_read_trylock(mm)) return; - vma = find_vma(mm, start); + + mas_set(&mas, start); + vma = mas_walk(&mas); if (!vma) { reset_ptenuma_scan(p); + mas_reset(&mas); start = 0; - vma = mm->mmap; + mas_set(&mas, start); } - for (; vma; vma = vma->vm_next) { + + mas_for_each(&mas, vma, ULONG_MAX) { if (!vma_migratable(vma) || !vma_policy_mof(vma) || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { continue; diff --git a/kernel/sys.c b/kernel/sys.c index a730c03ee607..fe061c54a5a4 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1858,9 +1858,10 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd) err = -EBUSY; if (exe_file) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!vma->vm_file) continue; if (path_equal(&vma->vm_file->f_path, diff --git a/mm/gup.c b/mm/gup.c index b06e2a6d9018..20392efb2d9b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1465,6 +1465,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) struct vm_area_struct *vma = NULL; int locked = 0; long ret = 0; + MA_STATE(mas, &mm->mm_mt, start, start); end = start + len; @@ -1476,10 +1477,10 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) if (!locked) { locked = 1; mmap_read_lock(mm); - vma = find_vma(mm, nstart); + vma = mas_find(&mas, end); } else if (nstart >= vma->vm_end) - vma = vma->vm_next; - if (!vma || vma->vm_start >= end) + vma = mas_next(&mas, end); + if (!vma) break; /* * Set [nstart; nend) to intersection of desired address diff --git a/mm/huge_memory.c b/mm/huge_memory.c index ec2bb93f7431..c2627619d622 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2304,12 +2304,12 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_address(vma, end, false, NULL); /* - * If we're also updating the vma->vm_next->vm_start, if the new - * vm_next->vm_start isn't hpage aligned and it could previously + * If we're also updating the vma_next(vma)->vm_start, if the new + * vma_next()->vm_start isn't hpage aligned and it could previously * contain an hugepage: check if we need to split an huge pmd. */ if (adjust_next > 0) { - struct vm_area_struct *next = vma->vm_next; + struct vm_area_struct *next = vma_next(vma); unsigned long nstart = next->vm_start; nstart += adjust_next; if (nstart & ~HPAGE_PMD_MASK && diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 4e3dff13eb70..b20228f10725 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2049,6 +2049,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, struct mm_struct *mm; struct vm_area_struct *vma; int progress = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); VM_BUG_ON(!pages); lockdep_assert_held(&khugepaged_mm_lock); @@ -2070,13 +2071,14 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, * the next mm on the list. */ vma = NULL; + mas_set(&mas, khugepaged_scan.address); if (unlikely(!mmap_read_trylock(mm))) goto breakouterloop_mmap_lock; if (likely(!khugepaged_test_exit(mm))) - vma = find_vma(mm, khugepaged_scan.address); + vma = mas_find(&mas, ULONG_MAX); progress++; - for (; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { unsigned long hstart, hend; cond_resched(); diff --git a/mm/ksm.c b/mm/ksm.c index 0960750bb316..2e9e685c588f 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -968,6 +968,7 @@ static int unmerge_and_remove_all_rmap_items(void) struct mm_struct *mm; struct vm_area_struct *vma; int err = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); spin_lock(&ksm_mmlist_lock); ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next, @@ -978,7 +979,7 @@ static int unmerge_and_remove_all_rmap_items(void) mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { mm = mm_slot->mm; mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (ksm_test_exit(mm)) break; if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) @@ -2229,6 +2230,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) struct vm_area_struct *vma; struct rmap_item *rmap_item; int nid; + MA_STATE(mas, &mm->mm_mt, 0, 0); if (list_empty(&ksm_mm_head.mm_list)) return NULL; @@ -2289,10 +2291,12 @@ next_mm: mmap_read_lock(mm); if (ksm_test_exit(mm)) vma = NULL; - else - vma = find_vma(mm, ksm_scan.address); + else { + mas_set(&mas, ksm_scan.address); + vma = mas_find(&mas, ULONG_MAX); + } - for (; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!(vma->vm_flags & VM_MERGEABLE)) continue; if (ksm_scan.address < vma->vm_start) diff --git a/mm/madvise.c b/mm/madvise.c index 13f5677b9322..0da3c9a5f5b8 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1151,7 +1151,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh if (start >= end) goto out; if (prev) - vma = prev->vm_next; + vma = vma_next(mm, prev); else /* madvise_remove dropped mmap_lock */ vma = find_vma(mm, start); } diff --git a/mm/memory.c b/mm/memory.c index c48f8df6e502..3b0fe38f967d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4903,8 +4903,8 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, * Check if this is a VM_IO | VM_PFNMAP VMA, which * we can access using slightly different code. */ - vma = find_vma(mm, addr); - if (!vma || vma->vm_start > addr) + vma = find_vma_intersection(mm, addr, addr + 1); + if (!vma) break; if (vma->vm_ops && vma->vm_ops->access) ret = vma->vm_ops->access(vma, addr, buf, diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 3ca4898f3f24..f16ff55b10c8 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -404,9 +404,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) + mas_for_each(&mas, vma, ULONG_MAX) mpol_rebind_policy(vma->vm_policy, new); mmap_write_unlock(mm); } @@ -671,7 +672,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma, static int queue_pages_test_walk(unsigned long start, unsigned long end, struct mm_walk *walk) { - struct vm_area_struct *vma = walk->vma; + struct vm_area_struct *next, *vma = walk->vma; struct queue_pages *qp = walk->private; unsigned long endvma = vma->vm_end; unsigned long flags = qp->flags; @@ -686,9 +687,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, /* hole at head side of range */ return -EFAULT; } + next = vma_next(vma->vm_mm, vma); if (!(flags & MPOL_MF_DISCONTIG_OK) && ((vma->vm_end < qp->end) && - (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) + (!next || vma->vm_end < next->vm_start))) /* hole at middle or tail of range */ return -EFAULT; @@ -809,21 +811,22 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, pgoff_t pgoff; unsigned long vmstart; unsigned long vmend; + MA_STATE(mas, &mm->mm_mt, start, start); - vma = find_vma(mm, start); + vma = mas_find(&mas, ULONG_MAX); VM_BUG_ON(!vma); - prev = vma->vm_prev; + prev = vma_mas_prev(&mas); if (start > vma->vm_start) prev = vma; - for (; vma && vma->vm_start < end; prev = vma, vma = next) { - next = vma->vm_next; + mas_for_each(&mas, vma, end) { + next = vma_next(mm, vma); vmstart = max(start, vma->vm_start); vmend = min(end, vma->vm_end); if (mpol_equal(vma_policy(vma), new_pol)) - continue; + goto next; pgoff = vma->vm_pgoff + ((vmstart - vma->vm_start) >> PAGE_SHIFT); @@ -832,7 +835,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, new_pol, vma->vm_userfaultfd_ctx); if (prev) { vma = prev; - next = vma->vm_next; + next = vma_next(mm, vma); if (mpol_equal(vma_policy(vma), new_pol)) continue; /* vma_merge() joined vma && vma->next, case 8 */ @@ -852,6 +855,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, err = vma_replace_policy(vma, new_pol); if (err) goto out; +next: + prev = vma; } out: @@ -1217,13 +1222,12 @@ static struct page *new_page(struct page *page, unsigned long start) { struct vm_area_struct *vma; unsigned long address; + MA_STATE(mas, ¤t->mm->mm_mt, start, start); - vma = find_vma(current->mm, start); - while (vma) { + mas_for_each(&mas, vma, ULONG_MAX) { address = page_address_in_vma(page, vma); if (address != -EFAULT) break; - vma = vma->vm_next; } if (PageHuge(page)) { diff --git a/mm/mlock.c b/mm/mlock.c index 884b1216da6a..c5337fbf7139 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -591,6 +591,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, unsigned long nstart, end, tmp; struct vm_area_struct * vma, * prev; int error; + MA_STATE(mas, ¤t->mm->mm_mt, start, start); VM_BUG_ON(offset_in_page(start)); VM_BUG_ON(len != PAGE_ALIGN(len)); @@ -599,11 +600,11 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, return -EINVAL; if (end == start) return 0; - vma = find_vma(current->mm, start); - if (!vma || vma->vm_start > start) + vma = mas_walk(&mas); + if (!vma) return -ENOMEM; - prev = vma->vm_prev; + prev = mas_prev(&mas, 0); if (start > vma->vm_start) prev = vma; @@ -625,7 +626,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, if (nstart >= end) break; - vma = prev->vm_next; + vma = vma_next(prev->vm_mm, prev); if (!vma || vma->vm_start != nstart) { error = -ENOMEM; break; @@ -646,15 +647,16 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, { struct vm_area_struct *vma; unsigned long count = 0; + MA_STATE(mas, &mm->mm_mt, start, start); if (mm == NULL) mm = current->mm; - vma = find_vma(mm, start); + vma = mas_find(&mas, ULONG_MAX); if (vma == NULL) vma = mm->mmap; - for (; vma ; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (start >= vma->vm_end) continue; if (start + len <= vma->vm_start) @@ -787,7 +789,7 @@ static int apply_mlockall_flags(int flags) to_add |= VM_LOCKONFAULT; } - for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { + for (vma = current->mm->mmap; vma ; vma = vma_next(vma->vm_mm, prev)) { vm_flags_t newflags; newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; diff --git a/mm/mmap.c b/mm/mmap.c index 3f7479e888e6..0b4680ef19c5 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -464,24 +464,6 @@ static bool range_has_overlap(struct mm_struct *mm, unsigned long start, return existing ? true : false; } -/* - * vma_next() - Get the next VMA. - * @mm: The mm_struct. - * @vma: The current vma. - * - * If @vma is NULL, return the first vma in the mm. - * - * Returns: The next VMA after @vma. - */ -static inline struct vm_area_struct *vma_next(struct mm_struct *mm, - struct vm_area_struct *vma) -{ - if (!vma) - return mm->mmap; - - return vma->vm_next; -} - static unsigned long count_vma_pages_range(struct mm_struct *mm, unsigned long addr, unsigned long end) { @@ -1096,6 +1078,24 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, return 0; } +/* + * vma_next_wrap() - Get the next VMA of the first. + * @mm: The mm_struct. + * @vma: The current vma. + * + * If @vma is NULL, return the first vma in the mm. + * + * Returns: The next VMA after @vma. + */ +static inline struct vm_area_struct *vma_next_wrap(struct mm_struct *mm, + struct vm_area_struct *vma) +{ + if (!vma) + return mm->mmap; + + return vma_next(mm, vma); +} + /* * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out * whether that can be merged with its predecessor or its successor. @@ -1157,7 +1157,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, if (vm_flags & VM_SPECIAL) return NULL; - next = vma_next(mm, prev); + next = vma_next_wrap(mm, prev); area = next; if (area && area->vm_end == end) /* cases 6, 7, 8 */ next = next->vm_next; diff --git a/mm/mprotect.c b/mm/mprotect.c index 56c02beb6041..5492833af73b 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -518,6 +518,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); + MA_STATE(mas, ¤t->mm->mm_mt, start, start); start = untagged_addr(start); @@ -549,11 +550,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; - vma = find_vma(current->mm, start); + vma = mas_walk(&mas); error = -ENOMEM; if (!vma) goto out; - prev = vma->vm_prev; + prev = vma_prev(vma->vm_mm, vma); if (unlikely(grows & PROT_GROWSDOWN)) { if (vma->vm_start >= end) goto out; @@ -626,7 +627,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (nstart >= end) goto out; - vma = prev->vm_next; + vma = mas_next(&mas, ULONG_MAX); if (!vma || vma->vm_start != nstart) { error = -ENOMEM; goto out; diff --git a/mm/mremap.c b/mm/mremap.c index a7526a8c1fe5..3b6e7f032463 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -465,7 +465,7 @@ out: if (excess) { vma->vm_flags |= VM_ACCOUNT; if (split) - vma->vm_next->vm_flags |= VM_ACCOUNT; + vma_next(mm, vma)->vm_flags |= VM_ACCOUNT; } return new_addr; @@ -638,9 +638,11 @@ out: static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) { unsigned long end = vma->vm_end + delta; + struct vm_area_struct *next; if (end < vma->vm_end) /* overflow */ return 0; - if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ + next = vma_next(vma->vm_mm, vma); + if (next && next->vm_start < end) /* intersection */ return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, 0, MAP_FIXED) & ~PAGE_MASK) diff --git a/mm/msync.c b/mm/msync.c index 69c6d2029531..8100ad5b12eb 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -100,7 +100,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) error = 0; goto out_unlock; } - vma = vma->vm_next; + vma = vma_next(mm, vma); } } out_unlock: diff --git a/mm/nommu.c b/mm/nommu.c index 0faf39b32cdb..4e5cc63728b6 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -677,6 +677,7 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* check the cache first */ vma = vmacache_find(mm, addr); @@ -685,7 +686,7 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) /* trawl the list (there may be multiple mappings in which addr * resides) */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (vma->vm_start > addr) return NULL; if (vma->vm_end > addr) { @@ -726,6 +727,7 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, { struct vm_area_struct *vma; unsigned long end = addr + len; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* check the cache first */ vma = vmacache_find_exact(mm, addr, end); @@ -734,7 +736,7 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, /* trawl the list (there may be multiple mappings in which addr * resides) */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (vma->vm_start < addr) continue; if (vma->vm_start > addr) @@ -1485,7 +1487,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list return -EINVAL; if (end == vma->vm_end) goto erase_whole_vma; - vma = vma->vm_next; + vma = vma_next(vma); } while (vma); return -EINVAL; } else { @@ -1543,7 +1545,7 @@ void exit_mmap(struct mm_struct *mm) mm->total_vm = 0; while ((vma = mm->mmap)) { - mm->mmap = vma->vm_next; + mm->mmap = vma_next(vma); delete_vma_from_mm(vma); delete_vma(mm, vma); cond_resched(); diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 8b84661a6410..a5e2045ec276 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -514,6 +514,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) { struct vm_area_struct *vma; bool ret = true; + MA_STATE(mas, &mm->mm_mt, 0, 0); /* * Tell all users of get_user/copy_from_user etc... that the content @@ -523,7 +524,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) */ set_bit(MMF_UNSTABLE, &mm->flags); - for (vma = mm->mmap ; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!can_madv_lru_vma(vma)) continue; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index e81640d9f177..20bd8d14d042 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -408,7 +408,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, } else { /* inside vma */ walk.vma = vma; next = min(end, vma->vm_end); - vma = vma->vm_next; + vma = vma_next(mm, vma);; err = walk_page_test(start, next, &walk); if (err > 0) { diff --git a/mm/swapfile.c b/mm/swapfile.c index d58361109066..10e1d60ff6f8 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2104,9 +2104,10 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type, { struct vm_area_struct *vma; int ret = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (vma->anon_vma) { ret = unuse_vma(vma, type, frontswap, fs_pages_to_unuse);