mm/uffd: Extract getting dst_pmd from mfill_atomic() into

uffd_get_dst_pmd()

Many memory types need to get the dst pmd, so move it to its own
function.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 8c3c9692c84cedb050bc4f3f771f87a3f8afffc8..8bc1c55005bfb73bc4b4b2600fa7acb49670182a 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -719,6 +719,37 @@ uffd_ctx_lock_and_validate_dst(struct userfaultfd_ctx *ctx,
        return 0;
 }
 
+static inline ssize_t
+uffd_get_dst_pmd(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma,
+               unsigned long dst_addr, pmd_t **dst_pmd)
+{
+       pmd_t dst_pmdval;
+
+       *dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);
+       if (unlikely(!*dst_pmd))
+               return -ENOMEM;
+
+       dst_pmdval = pmdp_get_lockless(*dst_pmd);
+       if (unlikely(pmd_none(dst_pmdval)) &&
+           unlikely(__pte_alloc(dst_mm, *dst_pmd)))
+               return -ENOMEM;
+
+       dst_pmdval = pmdp_get_lockless(*dst_pmd);
+       /*
+        * If the dst_pmd is THP don't override it and just be strict.
+        * (This includes the case where the PMD used to be THP and
+        * changed back to none after __pte_alloc().)
+        */
+       if (unlikely(!pmd_present(dst_pmdval) || pmd_trans_huge(dst_pmdval)))
+               return -EEXIST;
+
+       if (unlikely(pmd_bad(dst_pmdval)))
+               return -EFAULT;
+
+       return 0;
+}
+
+
 static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
                                            unsigned long dst_start,
                                            unsigned long src_start,
@@ -792,37 +823,12 @@ retry:
                goto out_unlock;
 
        while (src_addr < src_start + len) {
-               pmd_t dst_pmdval;
-
                VM_WARN_ON_ONCE(dst_addr >= dst_start + len);
 
-               dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);
-               if (unlikely(!dst_pmd)) {
-                       err = -ENOMEM;
+               err = uffd_get_dst_pmd(dst_mm, dst_vma, dst_addr, &dst_pmd);
+               if (err)
                        break;
-               }
 
-               dst_pmdval = pmdp_get_lockless(dst_pmd);
-               if (unlikely(pmd_none(dst_pmdval)) &&
-                   unlikely(__pte_alloc(dst_mm, dst_pmd))) {
-                       err = -ENOMEM;
-                       break;
-               }
-               dst_pmdval = pmdp_get_lockless(dst_pmd);
-               /*
-                * If the dst_pmd is THP don't override it and just be strict.
-                * (This includes the case where the PMD used to be THP and
-                * changed back to none after __pte_alloc().)
-                */
-               if (unlikely(!pmd_present(dst_pmdval) ||
-                               pmd_trans_huge(dst_pmdval))) {
-                       err = -EEXIST;
-                       break;
-               }
-               if (unlikely(pmd_bad(dst_pmdval))) {
-                       err = -EFAULT;
-                       break;
-               }
                /*
                 * For shmem mappings, khugepaged is allowed to remove page
                 * tables under us; pte_offset_map_lock() will deal with that.