mm/userfaultfd: Inline mfill_atomic into zeropage implementation

zeropage can use the normal mfill_atomic() loop, but inlining means that
the uffd_ops->op setup can be removed.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index 34539a142d388e694a3329bf23d6895b3c89eb64..c3e765dadfdb830c5e98757c6d83171b78119d65 100644 (file)
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -167,9 +167,8 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma,
 
 ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start,
                unsigned long src_start, unsigned long len, bool wp);
-extern ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx,
-                                    unsigned long dst_start,
-                                    unsigned long len);
+ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx,
+               unsigned long dst_start, unsigned long len);
 extern ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long dst_start,
                                     unsigned long len, uffd_flags_t flags);
 extern ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start,

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index e9b6e3e1882aee9e216c189d956312f0daf584e7..1d1b70f914d30c1c9ddcb72c5e82dbc31b827c90 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -688,8 +688,12 @@ out_unlock:
 
 static __always_inline ssize_t mfill_loop(struct uffd_info *info)
 {
-       ssize_t err = 0;
+       ssize_t err = -EINVAL;
+
+       if (!info->op)
+               goto out_unlock;
 
+       err = 0;
        while (!err && (info->src_addr < info->src_last)) {
                VM_WARN_ON_ONCE(info->dst_addr >= info->dst_last);
                err = info->op(info);
@@ -701,6 +705,7 @@ static __always_inline ssize_t mfill_loop(struct uffd_info *info)
                }
        }
 
+out_unlock:
        uffd_ctx_unlock(info);
        return err;
 }
@@ -798,11 +803,25 @@ out:
 }
 
 ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx,
-                             unsigned long start,
-                             unsigned long len)
+               unsigned long dst_start, unsigned long len)
 {
-       return mfill_atomic(ctx, start, 0, len,
-                           uffd_flags_set_mode(0, MFILL_ATOMIC_ZEROPAGE));
+       ssize_t err;
+       struct uffd_info info = UFFD_STRUCT_INIT(dst_start, 0, len, ctx,
+                                                false);
+
+       mfill_atomic_validate(&info);
+       err = mfill_setup(&info);
+       if (err)
+               goto out;
+
+       info.op = info.uffd_ops->zeropage;
+       err = mfill_loop(&info);
+
+out:
+       VM_WARN_ON_ONCE(info.copied < 0);
+       VM_WARN_ON_ONCE(err > 0);
+       VM_WARN_ON_ONCE(!info.copied && !err);
+       return info.copied ? info.copied : err;
 }
 
 ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long start,