From 2277259f140950a2a957af825e8c9653873342f8 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Wed, 29 Oct 2025 21:28:29 -0400 Subject: [PATCH] mm/userfaultfd: Inline mfill_atomic() into poison operation and remove uffd_flags_t The inline allows for the poison operation to know what uffd_ops to use for the operation due to call path. Drop the code that figures out uffd_ops. The uffd_flags_t existed to pass the operation data around and is no longer used. Signed-off-by: Liam R. Howlett --- fs/userfaultfd.c | 2 +- include/linux/userfaultfd_k.h | 26 +---------- mm/userfaultfd.c | 87 ++++++++--------------------------- 3 files changed, 22 insertions(+), 93 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 972c6f75b550..449e39df003f 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1842,7 +1842,7 @@ static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long if (mmget_not_zero(ctx->mm)) { ret = mfill_atomic_poison(ctx, uffdio_poison.range.start, - uffdio_poison.range.len, 0); + uffdio_poison.range.len); mmput(ctx->mm); } else { return -ESRCH; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 4154fad2af55..2c8b824ab4cc 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -80,9 +80,6 @@ struct userfaultfd_ctx { extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); -/* A combined operation mode + behavior flags. */ -typedef unsigned int __bitwise uffd_flags_t; - /* Mutually exclusive modes of operation. */ enum mfill_atomic_mode { MFILL_ATOMIC_COPY, @@ -142,25 +139,6 @@ struct vm_uffd_ops { void (*complete_register)(struct vm_area_struct *vma); }; -#define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) -#define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr)) -#define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr)) -#define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1)) - -static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected) -{ - return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected); -} - -static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode) -{ - flags &= ~MFILL_ATOMIC_MODE_MASK; - return flags | ((__force uffd_flags_t) mode); -} - -/* Flags controlling behavior. These behavior changes are mode-independent. */ -#define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) - int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, bool newly_allocated, bool wp); @@ -171,8 +149,8 @@ ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long len); ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long len, bool wp); -extern ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, - unsigned long len, uffd_flags_t flags); +ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len); extern int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, unsigned long len, bool enable_wp); extern long uffd_wp_range(struct vm_area_struct *vma, diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 0b4df1488811..bf55556f2463 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -594,40 +594,6 @@ uffd_get_dst_pmd(struct vm_area_struct *dst_vma, unsigned long dst_addr, return 0; } -static inline ssize_t uffd_info_set_ops(struct uffd_info *info, - uffd_flags_t flags) -{ - /* - * For shmem mappings, khugepaged is allowed to remove page - * tables under us; pte_offset_map_lock() will deal with that. - */ - if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) { - info->op = info->uffd_ops->poison; - } else if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) { - info->op = info->uffd_ops->cont; - } else if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - info->op = info->uffd_ops->copy; - /* - * The normal page fault path for a shmem will invoke - * the fault, fill the hole in the file and COW it right - * away. The result generates plain anonymous memory. So - * when we are asked to fill an hole in a MAP_PRIVATE - * shmem mapping, we'll generate anonymous memory - * directly without actually filling the hole. For the - * MAP_PRIVATE case the robustness check only happens in - * the pagetable (to verify it's still none) and not in - * the radix tree. - */ - } else { - info->op = info->uffd_ops->zeropage; - } - - if (!info->op) - return -EINVAL; - - return 0; -} - static inline ssize_t uffd_info_setup(struct uffd_info *info) { @@ -710,36 +676,6 @@ out_unlock: return err; } -static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, - unsigned long dst_start, - unsigned long src_start, - unsigned long len, - uffd_flags_t flags) -{ - ssize_t err; - struct uffd_info info = UFFD_STRUCT_INIT(dst_start, src_start, len, ctx, - flags & MFILL_ATOMIC_WP); - - mfill_atomic_validate(&info); - err = mfill_setup(&info); - if (err) - goto out; - - err = uffd_info_set_ops(&info, flags); - if (err) { - uffd_ctx_unlock(&info); - goto out; - } - - err = mfill_loop(&info); - -out: - VM_WARN_ON_ONCE(info.copied < 0); - VM_WARN_ON_ONCE(err > 0); - VM_WARN_ON_ONCE(!info.copied && !err); - return info.copied ? info.copied : err; -} - static __always_inline ssize_t mfill_copy_loop(struct uffd_info *info) { ssize_t err = -EINVAL; @@ -853,11 +789,26 @@ out: return info.copied ? info.copied : err; } -ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, - unsigned long len, uffd_flags_t flags) +ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, + unsigned long dst_start, unsigned long len) { - return mfill_atomic(ctx, start, 0, len, - uffd_flags_set_mode(flags, MFILL_ATOMIC_POISON)); + ssize_t err; + struct uffd_info info = UFFD_STRUCT_INIT(dst_start, 0, len, ctx, + false); + + mfill_atomic_validate(&info); + err = mfill_setup(&info); + if (err) + goto out; + + info.op = info.uffd_ops->poison; + err = mfill_loop(&info); + +out: + VM_WARN_ON_ONCE(info.copied < 0); + VM_WARN_ON_ONCE(err > 0); + VM_WARN_ON_ONCE(!info.copied && !err); + return info.copied ? info.copied : err; } long uffd_wp_range(struct vm_area_struct *dst_vma, -- 2.51.0