s390: Use inline qualifier for all EX_TABLE and ALTERNATIVE inline assemblies

Use asm_inline for all inline assemblies which make use of the EX_TABLE or
ALTERNATIVE macros.

These macros expand to many lines and the compiler assumes the number of
lines within an inline assembly is the same as the number of instructions
within an inline assembly. This has an effect on inlining and loop
unrolling decisions.

In order to avoid incorrect assumptions use asm_inline, which tells the
compiler that an inline assembly has the smallest possible size.

In order to avoid confusion when asm_inline should be used or not, since a
couple of inline assemblies are quite large: the rule is to always use
asm_inline whenever the EX_TABLE or ALTERNATIVE macro is used. In specific
cases there may be reasons to not follow this guideline, but that should
be documented with the corresponding code.

Using the inline qualifier everywhere has only a small effect on the kernel
image size:

add/remove: 0/10 grow/shrink: 19/8 up/down: 1492/-1858 (-366)

The only location where this seems to matter is load_unaligned_zeropad()
from word-at-a-time.h where the compiler inlines more functions within the
dcache code, which is indeed code where performance matters.

Suggested-by: Juergen Christ <jchrist@linux.ibm.com>
Reviewed-by: Juergen Christ <jchrist@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c
index b49c906b57dc037db72790fef9f34094c76c25e3..d04e9b89d14a669713a6603f9a497a6d09763b56 100644 (file)
--- a/arch/s390/boot/ipl_parm.c
+++ b/arch/s390/boot/ipl_parm.c
@@ -37,7 +37,7 @@ static inline int __diag308(unsigned long subcode, void *addr)
 {
        union register_pair r1 = { .even = (unsigned long)addr, .odd = 0 };
 
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[r1],%[subcode],0x308\n"
                "0:\n"
                EX_TABLE(0b, 0b)

diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c
index eb1badb93c40b25117d39a56e5ac1ac7ae484ffd..45e3d057cfaa31624fde8da04af7256084552eb7 100644 (file)
--- a/arch/s390/boot/physmem_info.c
+++ b/arch/s390/boot/physmem_info.c
@@ -67,7 +67,7 @@ static int __diag260(unsigned long rx1, unsigned long rx2)
        rx.odd  = rx2;
        ry = 0x10; /* storage configuration */
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[rx],%[ry],0x260\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -105,7 +105,7 @@ static int diag500_storage_limit(unsigned long *max_physmem_end)
 {
        unsigned long storage_limit;
 
-       asm volatile(
+       asm_inline volatile(
                "       lghi    %%r1,%[subcode]\n"
                "       lghi    %%r2,0\n"
                "       diag    %%r2,%%r4,0x500\n"
@@ -126,7 +126,7 @@ static int tprot(unsigned long addr)
        int cc, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       tprot   0(%[addr]),0\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"

diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
index 099b2327c781f60ec271908310ca67edc3537949..06316fb8e0fad8c55b02eb91349c171d9a1c4e70 100644 (file)
--- a/arch/s390/boot/startup.c
+++ b/arch/s390/boot/startup.c
@@ -77,7 +77,7 @@ static void detect_diag9c(void)
        int rc = 1;
 
        cpu = stap();
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[cpu],%%r0,0x9c\n"
                "0:     lhi     %[rc],0\n"
                "1:\n"
@@ -138,7 +138,7 @@ static int cmma_test_essa(void)
        int rc = 1;
 
        /* Test ESSA_GET_STATE */
-       asm volatile(
+       asm_inline volatile(
                "       .insn   rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n"
                "0:     lhi     %[rc],0\n"
                "1:\n"

diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h
index e1a279e0d6a616cd198d57b55f9a4eca81d0b47f..1798fbd59068bab106b63c2caacc91ab9f79d04c 100644 (file)
--- a/arch/s390/include/asm/cpu_mf.h
+++ b/arch/s390/include/asm/cpu_mf.h
@@ -171,7 +171,7 @@ static inline int qctri(struct cpumf_ctr_info *info)
 {
        int rc = -EINVAL;
 
-       asm volatile (
+       asm_inline volatile (
                "0:     qctri   %1\n"
                "1:     lhi     %0,0\n"
                "2:\n"
@@ -185,7 +185,7 @@ static inline int lcctl(u64 ctl)
 {
        int cc;
 
-       asm volatile (
+       asm_inline volatile (
                "       lcctl   %[ctl]\n"
                CC_IPM(cc)
                : CC_OUT(cc, cc)
@@ -200,7 +200,7 @@ static inline int __ecctr(u64 ctr, u64 *content)
        u64 _content;
        int cc;
 
-       asm volatile (
+       asm_inline volatile (
                "       ecctr   %[_content],%[ctr]\n"
                CC_IPM(cc)
                : CC_OUT(cc, cc), [_content] "=d" (_content)

diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h
index 5790630e31f0212d8ee3a3557af9d0fb5ecb3540..8db8db3b10182ad77de684a8687b83d6a9d5bf73 100644 (file)
--- a/arch/s390/include/asm/diag.h
+++ b/arch/s390/include/asm/diag.h
@@ -66,7 +66,7 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn)
        end_addr = pfn_to_phys(start_pfn + num_pfn - 1);
 
        diag_stat_inc(DIAG_STAT_X010);
-       asm volatile(
+       asm_inline volatile(
                "0:     diag    %0,%1,0x10\n"
                "1:     nopr    %%r7\n"
                EX_TABLE(0b, 1b)

diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 4f8d5592c2981c52a00111254daf6b7a1db5c5b9..d09a92db95f70be9ca1973676d6a84abcba70188 100644 (file)
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -416,7 +416,11 @@ static __always_inline bool regs_irqs_disabled(struct pt_regs *regs)
 
 static __always_inline void bpon(void)
 {
-       asm volatile(ALTERNATIVE("nop", ".insn  rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)));
+       asm_inline volatile(
+               ALTERNATIVE("   nop\n",
+                           "   .insn   rrf,0xb2e80000,0,0,13,0\n",
+                           ALT_SPEC(82))
+               );
 }
 
 #endif /* __ASSEMBLY__ */

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index f30ee6ddfece244a3b65685ac748c4c7e7a5a60f..8629d70ec38b5dc767a6a9a42aad67aab22f74c4 100644 (file)
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -147,7 +147,7 @@ __put_user_##type##_noinstr(unsigned type __user *to,                       \
 {                                                                      \
        int rc;                                                         \
                                                                        \
-       asm volatile(                                                   \
+       asm_inline volatile(                                            \
                "       llilh   %%r0,%[spec]\n"                         \
                "0:     mvcos   %[to],%[from],%[size]\n"                \
                "1:     lhi     %[rc],0\n"                              \
@@ -263,7 +263,7 @@ __get_user_##type##_noinstr(unsigned type *to,                              \
 {                                                                      \
        int rc;                                                         \
                                                                        \
-       asm volatile(                                                   \
+       asm_inline volatile(                                            \
                "       lhi     %%r0,%[spec]\n"                         \
                "0:     mvcos   %[to],%[from],%[size]\n"                \
                "1:     lhi     %[rc],0\n"                              \
@@ -490,7 +490,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
                _old = ((unsigned int)old & 0xff) << shift;
                _new = ((unsigned int)new & 0xff) << shift;
                mask = ~(0xff << shift);
-               asm volatile(
+               asm_inline volatile(
                        "       spka    0(%[key])\n"
                        "       sacf    256\n"
                        "       llill   %[count],%[max_loops]\n"
@@ -538,7 +538,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
                _old = ((unsigned int)old & 0xffff) << shift;
                _new = ((unsigned int)new & 0xffff) << shift;
                mask = ~(0xffff << shift);
-               asm volatile(
+               asm_inline volatile(
                        "       spka    0(%[key])\n"
                        "       sacf    256\n"
                        "       llill   %[count],%[max_loops]\n"
@@ -580,7 +580,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
        case 4: {
                unsigned int prev = old;
 
-               asm volatile(
+               asm_inline volatile(
                        "       spka    0(%[key])\n"
                        "       sacf    256\n"
                        "0:     cs      %[prev],%[new],%[address]\n"
@@ -601,7 +601,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
        case 8: {
                unsigned long prev = old;
 
-               asm volatile(
+               asm_inline volatile(
                        "       spka    0(%[key])\n"
                        "       sacf    256\n"
                        "0:     csg     %[prev],%[new],%[address]\n"
@@ -622,7 +622,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
        case 16: {
                __uint128_t prev = old;
 
-               asm volatile(
+               asm_inline volatile(
                        "       spka    0(%[key])\n"
                        "       sacf    256\n"
                        "0:     cdsg    %[prev],%[new],%[address]\n"

diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
index 203acd6e431b12651af06f3234a8587c7bc4dc20..eaa19dee7699415b4d27236be3bb4224f5806155 100644 (file)
--- a/arch/s390/include/asm/word-at-a-time.h
+++ b/arch/s390/include/asm/word-at-a-time.h
@@ -52,7 +52,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 {
        unsigned long data;
 
-       asm volatile(
+       asm_inline volatile(
                "0:     lg      %[data],0(%[addr])\n"
                "1:     nopr    %%r7\n"
                EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])

diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c
index bf983513dd333bb674065b2fe2f39fdbde311a39..03f3a1e524305a3026a442fe10c610922c88fc26 100644 (file)
--- a/arch/s390/kernel/cert_store.c
+++ b/arch/s390/kernel/cert_store.c
@@ -235,7 +235,7 @@ static int __diag320(unsigned long subcode, void *addr)
 {
        union register_pair rp = { .even = (unsigned long)addr, };
 
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[rp],%[subcode],0x320\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b, 0b)

diff --git a/arch/s390/kernel/diag/diag.c b/arch/s390/kernel/diag/diag.c
index e15b8dee3228dbbe0ad6d8518f009cecfb78acc9..56b862ba9be849393b022412f2ffe17677ddb925 100644 (file)
--- a/arch/s390/kernel/diag/diag.c
+++ b/arch/s390/kernel/diag/diag.c
@@ -195,7 +195,7 @@ static inline int __diag204(unsigned long *subcode, unsigned long size, void *ad
 {
        union register_pair rp = { .even = *subcode, .odd = size };
 
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[addr],%[rp],0x204\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b,0b)
@@ -286,7 +286,7 @@ int diag224(void *ptr)
        int rc = -EOPNOTSUPP;
 
        diag_stat_inc(DIAG_STAT_X224);
-       asm volatile("\n"
+       asm_inline volatile("\n"
                "       diag    %[type],%[addr],0x224\n"
                "0:     lhi     %[rc],0\n"
                "1:\n"

diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 8990e2dffcdf77fe4a5841fc6c27d42717287458..3b9d9ccfad63ef0bbcb7daadb973946006054da9 100644 (file)
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -186,7 +186,7 @@ static inline int __diag308(unsigned long subcode, unsigned long addr)
 
        r1.even = addr;
        r1.odd  = 0;
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[r1],%[subcode],0x308\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b,0b)

diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index 4daa3c76e999163a185153d304f0b90d1cb72a13..9705929d256f41f802d3aaed61ccabfa8461831f 100644 (file)
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -257,7 +257,7 @@ static void __init test_monitor_call(void)
 
        if (!IS_ENABLED(CONFIG_BUG))
                return;
-       asm volatile(
+       asm_inline volatile(
                "       mc      0,0\n"
                "0:     xgr     %0,%0\n"
                "1:\n"

diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c
index 1aac13bb8f53125af01c64ce27b5c3b11fc7b0e7..b01e73f5b9b8110680ec7ee9838d5402d61a1d1f 100644 (file)
--- a/arch/s390/mm/pfault.c
+++ b/arch/s390/mm/pfault.c
@@ -56,7 +56,7 @@ int __pfault_init(void)
        if (pfault_disable)
                return rc;
        diag_stat_inc(DIAG_STAT_X258);
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[refbk],%[rc],0x258\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b, 0b)
@@ -78,7 +78,7 @@ void __pfault_fini(void)
        if (pfault_disable)
                return;
        diag_stat_inc(DIAG_STAT_X258);
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[refbk],0,0x258\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b, 0b)

diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
index 14bf7e8d06b7a75dc8820e47d46a5f6e260045c6..bf083176076d54dde86b701576b0b20d8e2c5fc5 100644 (file)
--- a/arch/s390/pci/pci_clp.c
+++ b/arch/s390/pci/pci_clp.c
@@ -56,7 +56,7 @@ static inline int clp_get_ilp(unsigned long *ilp)
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rrf,0xb9a00000,%[mask],%[cmd],8,0\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -79,7 +79,7 @@ static __always_inline int clp_req(void *data, unsigned int lps)
        u64 ignored;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"

diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c
index f5a75ea7629afff04d85aa00b8bfa4d847a65f6c..eb978c8012be6bf694ac96116ef4991317df8151 100644 (file)
--- a/arch/s390/pci/pci_insn.c
+++ b/arch/s390/pci/pci_insn.c
@@ -160,7 +160,7 @@ static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status)
        u64 __data;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rre,0xb9d20000,%[data],%[req_off]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -229,7 +229,7 @@ static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status)
        u64 __data;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rre,0xb9d60000,%[data],%[ioaddr_len]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -267,7 +267,7 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rre,0xb9d00000,%[data],%[req_off]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -321,7 +321,7 @@ static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status)
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rre,0xb9d40000,%[data],%[ioaddr_len]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -356,7 +356,7 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rsy,0xeb00000000d0,%[req],%[offset],%[data]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"
@@ -410,7 +410,7 @@ static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status)
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"

diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c
index 46f99dc164ade4ca10f170cd66bdb648f92aa904..dc162c9951dd5351b0c642c93b1cfdca9fad0454 100644 (file)
--- a/arch/s390/pci/pci_mmio.c
+++ b/arch/s390/pci/pci_mmio.c
@@ -34,7 +34,7 @@ static inline int __pcistb_mio_inuser(
        int cc, exception;
 
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       sacf    256\n"
                "0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
                "1:     lhi     %[exc],0\n"
@@ -64,7 +64,7 @@ static inline int __pcistg_mio_inuser(
         * address space. pcistg then uses the user mappings.
         */
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       sacf    256\n"
                "0:     llgc    %[tmp],0(%[src])\n"
                "4:     sllg    %[val],%[val],8\n"
@@ -211,7 +211,7 @@ static inline int __pcilg_mio_inuser(
         * user address @dst
         */
        exception = 1;
-       asm volatile (
+       asm_inline volatile (
                "       sacf    256\n"
                "0:     .insn   rre,0xb9d60000,%[val],%[ioaddr_len]\n"
                "1:     lhi     %[exc],0\n"

diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c
index 8b39803cf94c976db7336fe4079b8d58b76d5059..56f1af8a7dddb81dfa6267ab0b275e4ac2db5049 100644 (file)
--- a/drivers/s390/block/dasd_diag.c
+++ b/drivers/s390/block/dasd_diag.c
@@ -76,7 +76,7 @@ static inline int __dia250(void *iob, int cmd)
        } addr_type;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[rx],%[cmd],0x250\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"

diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c
index 9418a9270d03ab7ccf2cadf43f2b5fab1baa301a..711f6982438e97ab126d4e1bae612bacfbc79c86 100644 (file)
--- a/drivers/s390/char/diag_ftp.c
+++ b/drivers/s390/char/diag_ftp.c
@@ -106,7 +106,7 @@ static int diag_ftp_2c4(struct diag_ftp_ldfpl *fpl,
        int rc;
 
        diag_stat_inc(DIAG_STAT_X2C4);
-       asm volatile(
+       asm_inline volatile(
                "       diag    %[addr],%[cmd],0x2c4\n"
                "0:     j       2f\n"
                "1:     la      %[rc],%[err]\n"

diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h
index f7ca6e2f6d18f892ccbedb73217fac9e4f51eb1a..b31a680e0871475b66139c683f8c89643ec2c4e8 100644 (file)
--- a/drivers/s390/char/sclp.h
+++ b/drivers/s390/char/sclp.h
@@ -318,7 +318,7 @@ static inline int sclp_service_call(sclp_cmdw_t command, void *sccb)
        int cc, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "0:     .insn   rre,0xb2200000,%[cmd],%[sccb]\n" /* servc */
                "1:     lhi     %[exc],0\n"
                "2:\n"

diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c
index 5ff1e51cddf399dc431b9ad2b44d31df4f959fd3..a540045b64a6ef733a7b96a5c52716762c6063d0 100644 (file)
--- a/drivers/s390/cio/ioasm.c
+++ b/drivers/s390/cio/ioasm.c
@@ -22,7 +22,7 @@ static inline int __stsch(struct subchannel_id schid, struct schib *addr)
        int ccode, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       lgr     1,%[r1]\n"
                "       stsch   %[addr]\n"
                "0:     lhi     %[exc],0\n"
@@ -52,7 +52,7 @@ static inline int __msch(struct subchannel_id schid, struct schib *addr)
        int ccode, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       lgr     1,%[r1]\n"
                "       msch    %[addr]\n"
                "0:     lhi     %[exc],0\n"
@@ -106,7 +106,7 @@ static inline int __ssch(struct subchannel_id schid, union orb *addr)
        int ccode, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       lgr     1,%[r1]\n"
                "       ssch    %[addr]\n"
                "0:     lhi     %[exc],0\n"
@@ -178,7 +178,7 @@ int chsc(void *chsc_area)
        int cc, exception;
 
        exception = 1;
-       asm volatile(
+       asm_inline volatile(
                "       .insn   rre,0xb25f0000,%[chsc_area],0\n"
                "0:     lhi     %[exc],0\n"
                "1:\n"