gup: avoid multiple user access locking/unlocking in fault_in_{read/write}able

fault_in_readable() and fault_in_writeable() perform __get_user() and
__put_user() in a loop, implying multiple user access locking/unlocking.

To avoid that, use user access blocks.

Link: https://lkml.kernel.org/r/720dcf79314acca1a78fae56d478cc851952149d.1637084492.git.christophe.leroy@csgroup.eu
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Andreas Gruenbacher <agruenba@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/gup.c b/mm/gup.c
index 2c51e9748a6a584f4e9d0c96f3f175fab6d89c17..be2a41feec7d7b85c558d3d9b7e19a7ea8b272c3 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1672,21 +1672,22 @@ size_t fault_in_writeable(char __user *uaddr, size_t size)
 
        if (unlikely(size == 0))
                return 0;
+       if (!user_write_access_begin(uaddr, size))
+               return size;
        if (!PAGE_ALIGNED(uaddr)) {
-               if (unlikely(__put_user(0, uaddr) != 0))
-                       return size;
+               unsafe_put_user(0, uaddr, out);
                uaddr = (char __user *)PAGE_ALIGN((unsigned long)uaddr);
        }
        end = (char __user *)PAGE_ALIGN((unsigned long)start + size);
        if (unlikely(end < start))
                end = NULL;
        while (uaddr != end) {
-               if (unlikely(__put_user(0, uaddr) != 0))
-                       goto out;
+               unsafe_put_user(0, uaddr, out);
                uaddr += PAGE_SIZE;
        }
 
 out:
+       user_write_access_end();
        if (size > uaddr - start)
                return size - (uaddr - start);
        return 0;
@@ -1771,21 +1772,22 @@ size_t fault_in_readable(const char __user *uaddr, size_t size)
 
        if (unlikely(size == 0))
                return 0;
+       if (!user_read_access_begin(uaddr, size))
+               return size;
        if (!PAGE_ALIGNED(uaddr)) {
-               if (unlikely(__get_user(c, uaddr) != 0))
-                       return size;
+               unsafe_get_user(c, uaddr, out);
                uaddr = (const char __user *)PAGE_ALIGN((unsigned long)uaddr);
        }
        end = (const char __user *)PAGE_ALIGN((unsigned long)start + size);
        if (unlikely(end < start))
                end = NULL;
        while (uaddr != end) {
-               if (unlikely(__get_user(c, uaddr) != 0))
-                       goto out;
+               unsafe_get_user(c, uaddr, out);
                uaddr += PAGE_SIZE;
        }
 
 out:
+       user_read_access_end();
        (void)c;
        if (size > uaddr - start)
                return size - (uaddr - start);