s390: add support for DCACHE_WORD_ACCESS

Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
speed up string operations in fs/dcache.c and fs/namei.c.

Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index ae29e4392664ad40236ade2614106fa2127d4a55..b0d67ac8695f9f232072c324a6b408c8f3b657bb 100644 (file)
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -131,6 +131,7 @@ config S390
        select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
        select BUILDTIME_TABLE_SORT
        select CLONE_BACKWARDS2
+       select DCACHE_WORD_ACCESS if !KMSAN
        select DMA_OPS if PCI
        select DYNAMIC_FTRACE if FUNCTION_TRACER
        select FUNCTION_ALIGNMENT_8B if CC_IS_GCC

diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
index d1ecc2b6e5d38c56cada24ee0575f8c53c369a01..4a6b0a8b6412f1f933c34a4003d5f32e9636618f 100644 (file)
--- a/arch/s390/include/asm/asm-extable.h
+++ b/arch/s390/include/asm/asm-extable.h
@@ -13,6 +13,7 @@
 #define EX_TYPE_UA_LOAD_MEM    4
 #define EX_TYPE_UA_LOAD_REG    5
 #define EX_TYPE_UA_LOAD_REGPAIR        6
+#define EX_TYPE_ZEROPAD                7
 
 #define EX_DATA_REG_ERR_SHIFT  0
 #define EX_DATA_REG_ERR                GENMASK(3, 0)
@@ -80,4 +81,7 @@
 #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero)   \
        __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
 
+#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr)          \
+       __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
+
 #endif /* __ASM_EXTABLE_H */

diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
index 09b37541a175b46c7454550ede83fce3acc976cd..2579f1694b82f4311a9e7faf721fad782426b4ff 100644 (file)
--- a/arch/s390/include/asm/word-at-a-time.h
+++ b/arch/s390/include/asm/word-at-a-time.h
@@ -3,6 +3,7 @@
 #define _ASM_WORD_AT_A_TIME_H
 
 #include <linux/kernel.h>
+#include <asm/asm-extable.h>
 #include <asm/bitsperlong.h>
 
 struct word_at_a_time {
@@ -39,4 +40,25 @@ static inline unsigned long zero_bytemask(unsigned long data)
        return ~1UL << data;
 }
 
+/*
+ * Load an unaligned word from kernel space.
+ *
+ * In the (very unlikely) case of the word being a page-crosser
+ * and the next page not being mapped, take the exception and
+ * return zeroes in the non-existing part.
+ */
+static inline unsigned long load_unaligned_zeropad(const void *addr)
+{
+       unsigned long data;
+
+       asm volatile(
+               "0:     lg      %[data],0(%[addr])\n"
+               "1:     nopr    %%r7\n"
+               EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
+               EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
+               : [data] "=d" (data)
+               : [addr] "a" (addr), "m" (*(unsigned long *)addr));
+       return data;
+}
+
 #endif /* _ASM_WORD_AT_A_TIME_H */

diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
index fe87291df95d3b69e5196816c6011da5b05fdf5f..0a0738a473af05e3149ebf2500ea0a79a3da5d73 100644 (file)
--- a/arch/s390/mm/extable.c
+++ b/arch/s390/mm/extable.c
@@ -61,6 +61,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex,
        return true;
 }
 
+static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+       unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
+       unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+       unsigned long data, addr, offset;
+
+       addr = regs->gprs[reg_addr];
+       offset = addr & (sizeof(unsigned long) - 1);
+       addr &= ~(sizeof(unsigned long) - 1);
+       data = *(unsigned long *)addr;
+       data <<= BITS_PER_BYTE * offset;
+       regs->gprs[reg_data] = data;
+       regs->psw.addr = extable_fixup(ex);
+       return true;
+}
+
 bool fixup_exception(struct pt_regs *regs)
 {
        const struct exception_table_entry *ex;
@@ -81,6 +97,8 @@ bool fixup_exception(struct pt_regs *regs)
                return ex_handler_ua_load_reg(ex, false, regs);
        case EX_TYPE_UA_LOAD_REGPAIR:
                return ex_handler_ua_load_reg(ex, true, regs);
+       case EX_TYPE_ZEROPAD:
+               return ex_handler_zeropad(ex, regs);
        }
        panic("invalid exception table entry");
 }