From 2bd43ffe01daeab6f920eb435109e821c82e1935 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 23 Oct 2025 15:10:04 -0400 Subject: [PATCH] mm/userfaultfd: Move hugetlb unlocks into hugetlb_mfill_atomic_pte() Instead of calling the unlock in the caller of uffd_ops->increment(), move the unlocks into the function itself. This moves the memory types closer to using the same mfill_atomic() implementation. Signed-off-by: Liam R. Howlett --- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 11 ++++++++--- mm/userfaultfd.c | 4 +--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2387513d6ae5..6f2f68c73d07 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -147,7 +147,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct folio **foliop); + struct folio **foliop, u32 hash); #endif /* CONFIG_USERFAULTFD */ long hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_desc *desc, vm_flags_t vm_flags); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f27ce58bf6cc..d59975c1e101 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6955,7 +6955,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct folio **foliop) + struct folio **foliop, + u32 hash) { struct mm_struct *dst_mm = dst_vma->vm_mm; bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE); @@ -6974,10 +6975,11 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) { ptl = huge_pte_lock(h, dst_mm, dst_pte); + ret = -EEXIST; /* Don't overwrite any existing PTEs (even markers) */ if (!huge_pte_none(huge_ptep_get(dst_mm, dst_addr, dst_pte))) { spin_unlock(ptl); - return -EEXIST; + goto out; } _dst_pte = make_pte_marker(PTE_MARKER_POISONED); @@ -6987,7 +6989,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, update_mmu_cache(dst_vma, dst_addr, dst_pte); spin_unlock(ptl); - return 0; + ret = 0; + goto out; } if (is_continue) { @@ -7155,6 +7158,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, folio_unlock(folio); ret = 0; out: + hugetlb_vma_unlock_read(dst_vma); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); return ret; out_release_unlock: spin_unlock(ptl); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index cf1722a62ae2..346b7b69b29e 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -634,10 +634,8 @@ retry: } err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, - src_addr, flags, &folio); + src_addr, flags, &folio, hash); - hugetlb_vma_unlock_read(dst_vma); - mutex_unlock(&hugetlb_fault_mutex_table[hash]); cond_resched(); -- 2.51.0