x86/uaccess: Improve the 8-byte getuser() case

Streamline the 8-byte case and drop the special handling. Use a macro
which hides the exception handling.

No functional changes.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/CAHk-=whYb2L_atsRk9pBiFiVLGe5wNZLHhRinA69yu6FiKvDsw@mail.gmail.com

diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index a1cb3a4e6742dfd35b13bfdc541e002ff96cbeab..a314622aa09346306db47689b4cb21e8d0313385 100644 (file)
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -44,21 +44,23 @@
        or %rdx, %rax
 .else
        cmp $TASK_SIZE_MAX-\size+1, %eax
-.if \size != 8
        jae .Lbad_get_user
-.else
-       jae .Lbad_get_user_8
-.endif
        sbb %edx, %edx          /* array_index_mask_nospec() */
        and %edx, %eax
 .endif
 .endm
 
+.macro UACCESS op src dst
+1:     \op \src,\dst
+       _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
+.endm
+
+
        .text
 SYM_FUNC_START(__get_user_1)
        check_range size=1
        ASM_STAC
-1:     movzbl (%_ASM_AX),%edx
+       UACCESS movzbl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -68,7 +70,7 @@ EXPORT_SYMBOL(__get_user_1)
 SYM_FUNC_START(__get_user_2)
        check_range size=2
        ASM_STAC
-2:     movzwl (%_ASM_AX),%edx
+       UACCESS movzwl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -78,7 +80,7 @@ EXPORT_SYMBOL(__get_user_2)
 SYM_FUNC_START(__get_user_4)
        check_range size=4
        ASM_STAC
-3:     movl (%_ASM_AX),%edx
+       UACCESS movl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -89,10 +91,11 @@ SYM_FUNC_START(__get_user_8)
        check_range size=8
        ASM_STAC
 #ifdef CONFIG_X86_64
-4:     movq (%_ASM_AX),%rdx
+       UACCESS movq (%_ASM_AX),%rdx
 #else
-4:     movl (%_ASM_AX),%edx
-5:     movl 4(%_ASM_AX),%ecx
+       xor %ecx,%ecx
+       UACCESS movl (%_ASM_AX),%edx
+       UACCESS movl 4(%_ASM_AX),%ecx
 #endif
        xor %eax,%eax
        ASM_CLAC
@@ -104,7 +107,7 @@ EXPORT_SYMBOL(__get_user_8)
 SYM_FUNC_START(__get_user_nocheck_1)
        ASM_STAC
        ASM_BARRIER_NOSPEC
-6:     movzbl (%_ASM_AX),%edx
+       UACCESS movzbl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -114,7 +117,7 @@ EXPORT_SYMBOL(__get_user_nocheck_1)
 SYM_FUNC_START(__get_user_nocheck_2)
        ASM_STAC
        ASM_BARRIER_NOSPEC
-7:     movzwl (%_ASM_AX),%edx
+       UACCESS movzwl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -124,7 +127,7 @@ EXPORT_SYMBOL(__get_user_nocheck_2)
 SYM_FUNC_START(__get_user_nocheck_4)
        ASM_STAC
        ASM_BARRIER_NOSPEC
-8:     movl (%_ASM_AX),%edx
+       UACCESS movl (%_ASM_AX),%edx
        xor %eax,%eax
        ASM_CLAC
        RET
@@ -135,10 +138,11 @@ SYM_FUNC_START(__get_user_nocheck_8)
        ASM_STAC
        ASM_BARRIER_NOSPEC
 #ifdef CONFIG_X86_64
-9:     movq (%_ASM_AX),%rdx
+       UACCESS movq (%_ASM_AX),%rdx
 #else
-9:     movl (%_ASM_AX),%edx
-10:    movl 4(%_ASM_AX),%ecx
+       xor %ecx,%ecx
+       UACCESS movl (%_ASM_AX),%edx
+       UACCESS movl 4(%_ASM_AX),%ecx
 #endif
        xor %eax,%eax
        ASM_CLAC
@@ -154,36 +158,3 @@ SYM_CODE_START_LOCAL(__get_user_handle_exception)
        mov $(-EFAULT),%_ASM_AX
        RET
 SYM_CODE_END(__get_user_handle_exception)
-
-#ifdef CONFIG_X86_32
-SYM_CODE_START_LOCAL(__get_user_8_handle_exception)
-       ASM_CLAC
-.Lbad_get_user_8:
-       xor %edx,%edx
-       xor %ecx,%ecx
-       mov $(-EFAULT),%_ASM_AX
-       RET
-SYM_CODE_END(__get_user_8_handle_exception)
-#endif
-
-/* get_user */
-       _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
-       _ASM_EXTABLE_UA(2b, __get_user_handle_exception)
-       _ASM_EXTABLE_UA(3b, __get_user_handle_exception)
-#ifdef CONFIG_X86_64
-       _ASM_EXTABLE_UA(4b, __get_user_handle_exception)
-#else
-       _ASM_EXTABLE_UA(4b, __get_user_8_handle_exception)
-       _ASM_EXTABLE_UA(5b, __get_user_8_handle_exception)
-#endif
-
-/* __get_user */
-       _ASM_EXTABLE_UA(6b, __get_user_handle_exception)
-       _ASM_EXTABLE_UA(7b, __get_user_handle_exception)
-       _ASM_EXTABLE_UA(8b, __get_user_handle_exception)
-#ifdef CONFIG_X86_64
-       _ASM_EXTABLE_UA(9b, __get_user_handle_exception)
-#else
-       _ASM_EXTABLE_UA(9b, __get_user_8_handle_exception)
-       _ASM_EXTABLE_UA(10b, __get_user_8_handle_exception)
-#endif