From: Liam R. Howlett Date: Mon, 27 Oct 2025 19:43:31 +0000 (-0400) Subject: mm/userfaultfd: Add failed_do_unlock(), move hugetlb implementation to X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=40da940030e6fe6614df254f6974c75eccd48686;p=users%2Fjedix%2Flinux-maple.git mm/userfaultfd: Add failed_do_unlock(), move hugetlb implementation to hugetlb.c Create and populate failed_do_unlock() in uffd ops. Moving the hugetlb implementation into hugetlb.c and calling it in the mfill_atomic_hugetlb() loop for now. The mfill_atomic() loop is also updated to use the function pointer of the types. Signed-off-by: Liam R. Howlett --- diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index f65073ae8d1c..d3b9d8adf32d 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -92,11 +92,19 @@ enum mfill_atomic_mode { NR_MFILL_ATOMIC_MODES, }; +/* VMA userfaultfd operations */ +ssize_t uffd_failed_do_unlock(struct userfaultfd_ctx *ctx, + struct vm_area_struct *dst, struct folio *folio, + unsigned long src_addr); + struct vm_uffd_ops { /* Required features below */ ssize_t (*is_dst_valid)(struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long len); unsigned long (*increment)(struct vm_area_struct *vma); + ssize_t (*failed_do_unlock)(struct userfaultfd_ctx *ctx, + struct vm_area_struct *dst, struct folio *folio, + unsigned long src_addr); }; #define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c7409c7cee28..bfd60f308272 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5513,9 +5513,26 @@ static inline unsigned long hugetlb_mfill_size(struct vm_area_struct *vma) static ssize_t hugetlb_is_dst_valid(struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long len); +static ssize_t hugetlb_failed_do_unlock(struct userfaultfd_ctx *ctx, + struct vm_area_struct *dst, struct folio *folio, + unsigned long src_addr) +{ + up_read(&ctx->map_changing_lock); +#ifdef CONFIG_PER_VMA_LOCK + vma_end_read(dst); +#else + mmap_read_unlock(dst->vm_mm); +#endif + + VM_WARN_ON_ONCE(!folio); + + return copy_folio_from_user(folio,(const void __user *)src_addr, true); +} + static const struct vm_uffd_ops hugetlb_uffd_ops = { .is_dst_valid = hugetlb_is_dst_valid, .increment = hugetlb_mfill_size, + .failed_do_unlock = hugetlb_failed_do_unlock, }; #endif diff --git a/mm/shmem.c b/mm/shmem.c index 385dd39f9337..86db6aa3b8e5 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -5210,6 +5210,7 @@ static int shmem_error_remove_folio(struct address_space *mapping, static const struct vm_uffd_ops shmem_uffd_ops = { .is_dst_valid = shmem_is_dst_valid, .increment = mfill_size, + .failed_do_unlock = uffd_failed_do_unlock, }; #endif diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index bc82a67d4782..47b169d25eb8 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -508,8 +508,9 @@ static ssize_t uffd_def_is_dst_valid(struct vm_area_struct *dst_vma, /* Anon vma ops */ static const struct vm_uffd_ops default_uffd_ops = { - .is_dst_valid = uffd_def_is_dst_valid, - .increment = mfill_size, + .is_dst_valid = uffd_def_is_dst_valid, + .increment = mfill_size, + .failed_do_unlock = uffd_failed_do_unlock, }; static inline const struct vm_uffd_ops *vma_get_uffd_ops(struct vm_area_struct *vma) @@ -520,6 +521,27 @@ static inline const struct vm_uffd_ops *vma_get_uffd_ops(struct vm_area_struct * return &default_uffd_ops; } +inline ssize_t +uffd_failed_do_unlock(struct userfaultfd_ctx *ctx, struct vm_area_struct *dst, + struct folio *folio, unsigned long src_addr) +{ + ssize_t err; + void *kaddr; + + up_read(&ctx->map_changing_lock); + uffd_mfill_unlock(dst); + VM_WARN_ON_ONCE(!folio); + + kaddr = kmap_local_folio(folio, 0); + err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kunmap_local(kaddr); + if (unlikely(err)) + return -EFAULT; + + flush_dcache_folio(folio); + return 0; +} + /* * Global checks live here, specific checks are in the callback. */ @@ -559,18 +581,6 @@ uffd_ctx_lock_and_validate_dst(struct userfaultfd_ctx *ctx, } #ifdef CONFIG_HUGETLB_PAGE -static -ssize_t hugetlb_failed_do_unlock(struct userfaultfd_ctx *ctx, - struct vm_area_struct *dst, struct folio *folio, - unsigned long src_addr) -{ - up_read(&ctx->map_changing_lock); - uffd_mfill_unlock(dst); - VM_WARN_ON_ONCE(!folio); - - return copy_folio_from_user(folio,(const void __user *)src_addr, true); -} - /* * mfill_atomic processing for HUGETLB vmas. Note that this routine is * called with either vma-lock or mmap_lock held, it will release the lock @@ -640,8 +650,8 @@ retry: cond_resched(); if (unlikely(err == -ENOENT)) { - err = hugetlb_failed_do_unlock(ctx, dst_vma, folio, - src_addr); + err = uffd_ops->failed_do_unlock(ctx, dst_vma, folio, + src_addr); if (unlikely(err)) { err = -EFAULT; goto out; @@ -685,27 +695,6 @@ extern ssize_t mfill_atomic_hugetlb(struct userfaultfd_ctx *ctx, uffd_flags_t flags); #endif /* CONFIG_HUGETLB_PAGE */ -static inline ssize_t -uffd_failed_do_unlock(struct userfaultfd_ctx *ctx, struct vm_area_struct *dst, - struct folio *folio, unsigned long src_addr) -{ - ssize_t err; - void *kaddr; - - up_read(&ctx->map_changing_lock); - uffd_mfill_unlock(dst); - VM_WARN_ON_ONCE(!folio); - - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); - kunmap_local(kaddr); - if (unlikely(err)) - return -EFAULT; - - flush_dcache_folio(folio); - return 0; -} - ssize_t uffd_get_dst_pmd(struct vm_area_struct *dst_vma, unsigned long dst_addr, pmd_t **dst_pmd) @@ -840,8 +829,8 @@ retry: cond_resched(); if (unlikely(err == -ENOENT)) { - err = uffd_failed_do_unlock(ctx, dst_vma, folio, - src_addr); + err = uffd_ops->failed_do_unlock(ctx, dst_vma, folio, + src_addr); if (unlikely(err)) goto out;