x86/putuser: Provide room for padding

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111146.746429822@infradead.org

diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S
index b7dfd60243b75c65f79e39a1486163a5d7f21ca6..32125224fccaadfb45863679bca5c5e7cba85009 100644 (file)
--- a/arch/x86/lib/putuser.S
+++ b/arch/x86/lib/putuser.S
@@ -47,8 +47,6 @@ SYM_FUNC_START(__put_user_1)
        LOAD_TASK_SIZE_MINUS_N(0)
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
-       ENDBR
        ASM_STAC
 1:     movb %al,(%_ASM_CX)
        xor %ecx,%ecx
@@ -56,54 +54,87 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
        RET
 SYM_FUNC_END(__put_user_1)
 EXPORT_SYMBOL(__put_user_1)
+
+SYM_FUNC_START(__put_user_nocheck_1)
+       ENDBR
+       ASM_STAC
+2:     movb %al,(%_ASM_CX)
+       xor %ecx,%ecx
+       ASM_CLAC
+       RET
+SYM_FUNC_END(__put_user_nocheck_1)
 EXPORT_SYMBOL(__put_user_nocheck_1)
 
 SYM_FUNC_START(__put_user_2)
        LOAD_TASK_SIZE_MINUS_N(1)
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
-       ENDBR
        ASM_STAC
-2:     movw %ax,(%_ASM_CX)
+3:     movw %ax,(%_ASM_CX)
        xor %ecx,%ecx
        ASM_CLAC
        RET
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
+
+SYM_FUNC_START(__put_user_nocheck_2)
+       ENDBR
+       ASM_STAC
+4:     movw %ax,(%_ASM_CX)
+       xor %ecx,%ecx
+       ASM_CLAC
+       RET
+SYM_FUNC_END(__put_user_nocheck_2)
 EXPORT_SYMBOL(__put_user_nocheck_2)
 
 SYM_FUNC_START(__put_user_4)
        LOAD_TASK_SIZE_MINUS_N(3)
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
-       ENDBR
        ASM_STAC
-3:     movl %eax,(%_ASM_CX)
+5:     movl %eax,(%_ASM_CX)
        xor %ecx,%ecx
        ASM_CLAC
        RET
 SYM_FUNC_END(__put_user_4)
 EXPORT_SYMBOL(__put_user_4)
+
+SYM_FUNC_START(__put_user_nocheck_4)
+       ENDBR
+       ASM_STAC
+6:     movl %eax,(%_ASM_CX)
+       xor %ecx,%ecx
+       ASM_CLAC
+       RET
+SYM_FUNC_END(__put_user_nocheck_4)
 EXPORT_SYMBOL(__put_user_nocheck_4)
 
 SYM_FUNC_START(__put_user_8)
        LOAD_TASK_SIZE_MINUS_N(7)
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
-       ENDBR
        ASM_STAC
-4:     mov %_ASM_AX,(%_ASM_CX)
+7:     mov %_ASM_AX,(%_ASM_CX)
 #ifdef CONFIG_X86_32
-5:     movl %edx,4(%_ASM_CX)
+8:     movl %edx,4(%_ASM_CX)
 #endif
        xor %ecx,%ecx
        ASM_CLAC
        RET
 SYM_FUNC_END(__put_user_8)
 EXPORT_SYMBOL(__put_user_8)
+
+SYM_FUNC_START(__put_user_nocheck_8)
+       ENDBR
+       ASM_STAC
+9:     mov %_ASM_AX,(%_ASM_CX)
+#ifdef CONFIG_X86_32
+10:    movl %edx,4(%_ASM_CX)
+#endif
+       xor %ecx,%ecx
+       ASM_CLAC
+       RET
+SYM_FUNC_END(__put_user_nocheck_8)
 EXPORT_SYMBOL(__put_user_nocheck_8)
 
 SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
@@ -117,6 +148,11 @@ SYM_CODE_END(.Lbad_put_user_clac)
        _ASM_EXTABLE_UA(2b, .Lbad_put_user_clac)
        _ASM_EXTABLE_UA(3b, .Lbad_put_user_clac)
        _ASM_EXTABLE_UA(4b, .Lbad_put_user_clac)
-#ifdef CONFIG_X86_32
        _ASM_EXTABLE_UA(5b, .Lbad_put_user_clac)
+       _ASM_EXTABLE_UA(6b, .Lbad_put_user_clac)
+       _ASM_EXTABLE_UA(7b, .Lbad_put_user_clac)
+       _ASM_EXTABLE_UA(9b, .Lbad_put_user_clac)
+#ifdef CONFIG_X86_32
+       _ASM_EXTABLE_UA(8b, .Lbad_put_user_clac)
+       _ASM_EXTABLE_UA(10b, .Lbad_put_user_clac)
 #endif