mm: allow __get_user_pages() callers to avoid triggering page faults.

Some callers of __get_userpages() would like to tentatively probe for
pages which may or may not be faulted in, in contexts in which faulting
is prohibited.  __get_user_pages() can return early before the fault is
complete, but cannot avoid the fault entirely.

This change introduces FOLL_NOFAULT, which causes an early return if a
fault would occur, without triggering a fault, and indicating (if the
caller requests it) that the call would block.

Orabug: 18412802
Signed-off-by: Nick Alcock <nick.alcock@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>
Reviewed-by: Chuck Anderson <chuck.anderson@oracle.com>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0755b9fd03a7d936e805efb71cfc1d371ddea0d6..0fe9b2c06aeadde395f8deb732ccacb05afd1834 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2035,6 +2035,7 @@ static inline struct page *follow_page(struct vm_area_struct *vma,
 #define FOLL_NUMA      0x200   /* force NUMA hinting page fault */
 #define FOLL_MIGRATION 0x400   /* wait for page to replace migration entry */
 #define FOLL_TRIED     0x800   /* a retry, previous pass started an IO */
+#define FOLL_NOFAULT   0x1000  /* fail rather than fault pages in */
 
 typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
                        void *data);

diff --git a/mm/gup.c b/mm/gup.c
index 6297f6bccfb1e42bedde7d69479eefbd52317b23..6dd07dc3eb8218f3d68b3fc3067bb966ca856231 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -265,11 +265,18 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
        unsigned int fault_flags = 0;
        int ret;
 
+       if (unlikely(*flags & FOLL_NOFAULT)) {
+               if (nonblocking)
+                       *nonblocking = 0;
+               return -EBUSY;
+       }
+
        /* For mm_populate(), just skip the stack guard page. */
        if ((*flags & FOLL_POPULATE) &&
                        (stack_guard_page_start(vma, address) ||
                         stack_guard_page_end(vma, address + PAGE_SIZE)))
                return -ENOENT;
+
        if (*flags & FOLL_WRITE)
                fault_flags |= FAULT_FLAG_WRITE;
        if (nonblocking)
@@ -398,11 +405,16 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
  * appropriate) must be called after the page is finished with, and
  * before put_page is called.
  *
- * If @nonblocking != NULL, __get_user_pages will not wait for disk IO
- * or mmap_sem contention, and if waiting is needed to pin all pages,
- * *@nonblocking will be set to 0.  Further, if @gup_flags does not
- * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in
- * this case.
+ * If FOLL_NOFAULT is set, pinning of pages will cease as soon as a page is
+ * encountered that needs to be faulted in.  (No attempt is made to see if
+ * further pages could be pinned without faulting: the caller must do that, if
+ * desired.)
+ *
+ * If @nonblocking != NULL, __get_user_pages will not wait for disk IO or
+ * mmap_sem contention, and if waiting is needed to pin all pages, or
+ * FOLL_NOFAULT is set and a fault is needed, *@nonblocking will be set to 0.
+ * Further, if @gup_flags does not include FOLL_NOWAIT, the mmap_sem will be
+ * released via up_read() in this case.
  *
  * A caller using such a combination of @nonblocking and @gup_flags
  * must therefore hold the mmap_sem for reading only, and recognize

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 271e4432734c376baf0bf4b8953a38e391ac011c..706c0d0d8d03adb1e5429be6c146fcbe9b848d97 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3439,6 +3439,10 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
                        if (pte)
                                spin_unlock(ptl);
+
+                       if (flags & FOLL_NOFAULT)
+                               return i;
+
                        ret = hugetlb_fault(mm, vma, vaddr,
                                (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
                        if (!(ret & VM_FAULT_ERROR))