mm: Switch vma_merge(), split_vma(), and __split_vma to vma iterator

Drop the vmi_* functions and transition all users to use the vma
iterator directly.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index b3249388696a38d176c4393d06129dffe6a487f9..e60f86d6b91ce2098e6542e40ac64defb0665855 100644 (file)
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -883,7 +883,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
                        continue;
                }
                new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
-               prev = vmi_vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
+               prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
                                 new_flags, vma->anon_vma,
                                 vma->vm_file, vma->vm_pgoff,
                                 vma_policy(vma),
@@ -1426,7 +1426,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
                vma_end = min(end, vma->vm_end);
 
                new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
-               prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
+               prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
                                 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
                                 vma_policy(vma),
                                 ((struct vm_userfaultfd_ctx){ ctx }),
@@ -1437,12 +1437,12 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
                        goto next;
                }
                if (vma->vm_start < start) {
-                       ret = vmi_split_vma(&vmi, mm, vma, start, 1);
+                       ret = split_vma(&vmi, vma, start, 1);
                        if (ret)
                                break;
                }
                if (vma->vm_end > end) {
-                       ret = vmi_split_vma(&vmi, mm, vma, end, 0);
+                       ret = split_vma(&vmi, vma, end, 0);
                        if (ret)
                                break;
                }
@@ -1606,7 +1606,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
                        uffd_wp_range(mm, vma, start, vma_end - start, false);
 
                new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
-               prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
+               prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
                                 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
                                 vma_policy(vma),
                                 NULL_VM_UFFD_CTX, anon_vma_name(vma));
@@ -1615,13 +1615,13 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
                        goto next;
                }
                if (vma->vm_start < start) {
-                       ret = vmi_split_vma(&vmi, mm, vma, start, 1);
+                       ret = split_vma(&vmi, vma, start, 1);
                        if (ret)
                                break;
                }
                if (vma->vm_end > end) {
                        vma_iter_set(&vmi, vma->vm_end);
-                       ret = vmi_split_vma(&vmi, mm, vma, end, 0);
+                       ret = split_vma(&vmi, vma, end, 0);
                        if (ret)
                                break;
                }

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 2a3405863cd67aa6689a3e310b635daeadfdcf83..cec0eccde16e2fca785676387e85e6c257fd43d2 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2721,22 +2721,16 @@ static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
 {
        return __vma_adjust(vma, start, end, pgoff, insert, NULL);
 }
-extern struct vm_area_struct *vma_merge(struct mm_struct *,
-       struct vm_area_struct *prev, unsigned long addr, unsigned long end,
-       unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
-       struct mempolicy *, struct vm_userfaultfd_ctx, struct anon_vma_name *);
-extern struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
+extern struct vm_area_struct *vma_merge(struct vma_iterator *vmi,
        struct mm_struct *, struct vm_area_struct *prev, unsigned long addr,
        unsigned long end, unsigned long vm_flags, struct anon_vma *,
        struct file *, pgoff_t, struct mempolicy *, struct vm_userfaultfd_ctx,
        struct anon_vma_name *);
 extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
-extern int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *,
-       struct vm_area_struct *, unsigned long addr, int new_below);
-extern int split_vma(struct mm_struct *, struct vm_area_struct *,
-       unsigned long addr, int new_below);
-extern int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *,
-               struct vm_area_struct *, unsigned long addr, int new_below);
+extern int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
+                      unsigned long addr, int new_below);
+extern int split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
+                        unsigned long addr, int new_below);
 extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
 extern void unlink_file_vma(struct vm_area_struct *);
 extern struct vm_area_struct *copy_vma(struct vm_area_struct **,

diff --git a/mm/madvise.c b/mm/madvise.c
index 98615628d7e870328ec3167404780ac7160dfe39..ff538549873a8028f93d77c84c0e3434b56eb160 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -153,7 +153,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
        }
 
        pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
-       *prev = vmi_vma_merge(&vmi, mm, *prev, start, end, new_flags,
+       *prev = vma_merge(&vmi, mm, *prev, start, end, new_flags,
                          vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
                          vma->vm_userfaultfd_ctx, anon_name);
        if (*prev) {
@@ -166,7 +166,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
        if (start != vma->vm_start) {
                if (unlikely(mm->map_count >= sysctl_max_map_count))
                        return -ENOMEM;
-               error = vmi__split_vma(&vmi, mm, vma, start, 1);
+               error = __split_vma(&vmi, vma, start, 1);
                if (error)
                        return error;
        }
@@ -174,7 +174,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
        if (end != vma->vm_end) {
                if (unlikely(mm->map_count >= sysctl_max_map_count))
                        return -ENOMEM;
-               error = vmi__split_vma(&vmi, mm, vma, end, 0);
+               error = __split_vma(&vmi, vma, end, 0);
                if (error)
                        return error;
        }

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 5bf4ab8eee494efadd2b2ea6f8543f142cd4ae17..ee9751323c5976bf492c331c2ca9f0803ab251d0 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -810,7 +810,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
 
                pgoff = vma->vm_pgoff +
                        ((vmstart - vma->vm_start) >> PAGE_SHIFT);
-               prev = vmi_vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
+               prev = vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
                                 vma->anon_vma, vma->vm_file, pgoff,
                                 new_pol, vma->vm_userfaultfd_ctx,
                                 anon_vma_name(vma));
@@ -819,12 +819,12 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
                        goto replace;
                }
                if (vma->vm_start != vmstart) {
-                       err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmstart, 1);
+                       err = split_vma(&vmi, vma, vmstart, 1);
                        if (err)
                                goto out;
                }
                if (vma->vm_end != vmend) {
-                       err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmend, 0);
+                       err = split_vma(&vmi, vma, vmend, 0);
                        if (err)
                                goto out;
                }

diff --git a/mm/mlock.c b/mm/mlock.c
index f06b02b631b5674aae43681ae51a2199eda946d9..393cddee2f06ea0a7f26e83bc4014e4b214e9c97 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -418,7 +418,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
                goto out;
 
        pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
-       *prev = vmi_vma_merge(vmi, mm, *prev, start, end, newflags,
+       *prev = vma_merge(vmi, mm, *prev, start, end, newflags,
                        vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
                        vma->vm_userfaultfd_ctx, anon_vma_name(vma));
        if (*prev) {
@@ -427,13 +427,13 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
        }
 
        if (start != vma->vm_start) {
-               ret = vmi_split_vma(vmi, mm, vma, start, 1);
+               ret = split_vma(vmi, vma, start, 1);
                if (ret)
                        goto out;
        }
 
        if (end != vma->vm_end) {
-               ret = vmi_split_vma(vmi, mm, vma, end, 0);
+               ret = split_vma(vmi, vma, end, 0);
                if (ret)
                        goto out;
        }

diff --git a/mm/mmap.c b/mm/mmap.c
index fcb852c34ef07528e34894d9f53b23e07219548d..b8a4b2bca3d30887e5252054830251f5b6fa20cb 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1072,7 +1072,7 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
  * parameter) may establish ptes with the wrong permissions of NNNN
  * instead of the right permissions of XXXX.
  */
-struct vm_area_struct *vma_merge(struct mm_struct *mm,
+struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
                        struct vm_area_struct *prev, unsigned long addr,
                        unsigned long end, unsigned long vm_flags,
                        struct anon_vma *anon_vma, struct file *file,
@@ -1081,7 +1081,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                        struct anon_vma_name *anon_name)
 {
        pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
-       struct vm_area_struct *mid, *next, *res;
+       struct vm_area_struct *mid, *next, *res = NULL;
        int err = -1;
        bool merge_prev = false;
        bool merge_next = false;
@@ -1147,26 +1147,11 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
        if (err)
                return NULL;
        khugepaged_enter_vma(res, vm_flags);
-       return res;
-}
 
-struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
-                       struct mm_struct *mm,
-                       struct vm_area_struct *prev, unsigned long addr,
-                       unsigned long end, unsigned long vm_flags,
-                       struct anon_vma *anon_vma, struct file *file,
-                       pgoff_t pgoff, struct mempolicy *policy,
-                       struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
-                       struct anon_vma_name *anon_name)
-{
-       struct vm_area_struct *tmp;
-
-       tmp = vma_merge(mm, prev, addr, end, vm_flags, anon_vma, file, pgoff,
-                       policy, vm_userfaultfd_ctx, anon_name);
-       if (tmp)
+       if (res)
                vma_iter_set(vmi, end);
 
-       return tmp;
+       return res;
 }
 
 /*
@@ -2286,12 +2271,14 @@ static void unmap_region(struct mm_struct *mm, struct maple_tree *mt,
  * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
  * has already been checked or doesn't make sense to fail.
  */
-int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
                unsigned long addr, int new_below)
 {
        struct vm_area_struct *new;
        int err;
-       validate_mm_mt(mm);
+       unsigned long end = vma->vm_end;
+
+       validate_mm_mt(vma->vm_mm);
 
        if (vma->vm_ops && vma->vm_ops->may_split) {
                err = vma->vm_ops->may_split(vma, addr);
@@ -2331,8 +2318,10 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
                err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
 
        /* Success. */
-       if (!err)
+       if (!err) {
+               vma_iter_set(vmi, end);
                return 0;
+       }
 
        /* Avoid vm accounting in close() operation */
        new->vm_start = new->vm_end;
@@ -2347,46 +2336,21 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
        mpol_put(vma_policy(new));
  out_free_vma:
        vm_area_free(new);
-       validate_mm_mt(mm);
+       validate_mm_mt(vma->vm_mm);
        return err;
 }
-int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
-                  struct vm_area_struct *vma, unsigned long addr, int new_below)
-{
-       int ret;
-       unsigned long end = vma->vm_end;
-
-       ret = __split_vma(mm, vma, addr, new_below);
-       if (!ret)
-               vma_iter_set(vmi, end);
-
-       return ret;
-}
 
 /*
  * Split a vma into two pieces at address 'addr', a new vma is allocated
  * either for the first part or the tail.
  */
-int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
              unsigned long addr, int new_below)
 {
-       if (mm->map_count >= sysctl_max_map_count)
+       if (vma->vm_mm->map_count >= sysctl_max_map_count)
                return -ENOMEM;
 
-       return __split_vma(mm, vma, addr, new_below);
-}
-
-int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
-                 struct vm_area_struct *vma, unsigned long addr, int new_below)
-{
-       int ret;
-       unsigned long end = vma->vm_end;
-
-       ret = split_vma(mm, vma, addr, new_below);
-       if (!ret)
-               vma_iter_set(vmi, end);
-
-       return ret;
+       return __split_vma(vmi, vma, addr, new_below);
 }
 
 static inline int munmap_sidetree(struct vm_area_struct *vma,
@@ -2446,7 +2410,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
                if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
                        goto map_count_exceeded;
 
-               error = vmi__split_vma(vmi, mm, vma, start, 0);
+               error = __split_vma(vmi, vma, start, 0);
                if (error)
                        goto start_split_failed;
 
@@ -2467,7 +2431,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
                if (next->vm_end > end) {
                        struct vm_area_struct *split;
 
-                       error = vmi__split_vma(vmi, mm, next, end, 1);
+                       error = __split_vma(vmi, next, end, 1);
                        if (error)
                                goto end_split_failed;
 
@@ -2748,9 +2712,10 @@ cannot_expand:
                 * vma again as we may succeed this time.
                 */
                if (unlikely(vm_flags != vma->vm_flags && prev)) {
-                       merge = vmi_vma_merge(&vmi, mm, prev, vma->vm_start,
-                               vma->vm_end, vma->vm_flags, NULL, vma->vm_file,
-                               vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
+                       merge = vma_merge(&vmi, mm, prev, vma->vm_start,
+                                   vma->vm_end, vma->vm_flags, NULL,
+                                   vma->vm_file, vma->vm_pgoff, NULL,
+                                   NULL_VM_UFFD_CTX, NULL);
                        if (merge) {
                                /*
                                 * ->mmap() can change vma->vm_file and fput
@@ -3297,7 +3262,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
        if (new_vma && new_vma->vm_start < addr + len)
                return NULL;    /* should never get here */
 
-       new_vma = vmi_vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
+       new_vma = vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
                            vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
                            vma->vm_userfaultfd_ctx, anon_vma_name(vma));
        if (new_vma) {

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 994e8d991e782d305e4ebb0303606962234d09c1..bd5c04974bbc8acfda411ee6a5aad266eda652d4 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -602,7 +602,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
         * First try to merge with previous and/or next vma.
         */
        pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
-       *pprev = vmi_vma_merge(vmi, mm, *pprev, start, end, newflags,
+       *pprev = vma_merge(vmi, mm, *pprev, start, end, newflags,
                           vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
                           vma->vm_userfaultfd_ctx, anon_vma_name(vma));
        if (*pprev) {
@@ -614,13 +614,13 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
        *pprev = vma;
 
        if (start != vma->vm_start) {
-               error = vmi_split_vma(vmi, mm, vma, start, 1);
+               error = split_vma(vmi, vma, start, 1);
                if (error)
                        goto fail;
        }
 
        if (end != vma->vm_end) {
-               error = vmi_split_vma(vmi, mm, vma, end, 0);
+               error = split_vma(vmi, vma, end, 0);
                if (error)
                        goto fail;
        }

diff --git a/mm/mremap.c b/mm/mremap.c
index 7302a71058ce377e77b42536b304e43bb69eeca9..cbafea7fe89548ca9b43be54a4442890cf2611f9 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -1033,7 +1033,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                         * with the next vma if it becomes adjacent to the expanded vma and
                         * otherwise compatible.
                         */
-                       vma = vmi_vma_merge(&vmi, mm, vma, extension_start,
+                       vma = vma_merge(&vmi, mm, vma, extension_start,
                                extension_end, vma->vm_flags, vma->anon_vma,
                                vma->vm_file, extension_pgoff, vma_policy(vma),
                                vma->vm_userfaultfd_ctx, anon_vma_name(vma));