MIPS: Consistently declare TLB functions

Since at least the beginning of the git era we've declared our TLB
exception handling functions inconsistently. They're actually functions,
but we declare them as arrays of u32 where each u32 is an encoded
instruction. This has always been the case for arch/mips/mm/tlbex.c, and
has also been true for arch/mips/kernel/traps.c since commit
86a1708a9d54 ("MIPS: Make tlb exception handler definitions and
declarations match.") which aimed for consistency but did so by
consistently making the our C code inconsistent with our assembly.

This is all usually harmless, but when using GCC 7 or newer to build a
kernel targeting microMIPS (ie. CONFIG_CPU_MICROMIPS=y) it becomes
problematic. With microMIPS bit 0 of the program counter indicates the
ISA mode. When bit 0 is zero instructions are decoded using the standard
MIPS32 or MIPS64 ISA. When bit 0 is one instructions are decoded using
microMIPS. This means that function pointers become odd - their least
significant bit is one for microMIPS code. We work around this in cases
where we need to access code using loads & stores with our
msk_isa16_mode() macro which simply clears bit 0 of the value it is
given:

  #define msk_isa16_mode(x) ((x) & ~0x1)

For example we do this for our TLB load handler in
build_r4000_tlb_load_handler():

  u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl);

We then write code to p, expecting it to be suitably aligned (our LEAF
macro aligns functions on 4 byte boundaries, so (ulong)handle_tlbl will
give a value one greater than a multiple of 4 - ie. the start of a
function on a 4 byte boundary, with the ISA mode bit 0 set).

This worked fine up to GCC 6, but GCC 7 & onwards is smart enough to
presume that handle_tlbl which we declared as an array of u32s must be
aligned sufficiently that bit 0 of its address will never be set, and as
a result optimize out msk_isa16_mode(). This leads to p having an
address with bit 0 set, and when we go on to attempt to store code at
that address we take an address error exception due to the unaligned
memory access.

This leads to an exception prior to the kernel having configured its own
exception handlers, so we jump to whatever handlers the bootloader
configured. In the case of QEMU this results in a silent hang, since it
has no useful general exception vector.

Fix this by consistently declaring our TLB-related functions as
functions. For handle_tlbl(), handle_tlbs() & handle_tlbm() we do this
in asm/tlbex.h & we make use of the existing declaration of
tlbmiss_handler_setup_pgd() in asm/mmu_context.h. Our TLB handler
generation code in arch/mips/mm/tlbex.c is adjusted to deal with these
definitions, in most cases simply by casting the function pointers to
u32 pointers.

This allows us to include asm/mmu_context.h in arch/mips/mm/tlbex.c to
get the definitions of tlbmiss_handler_setup_pgd & pgd_current, removing
some needless duplication. Consistently using msk_isa16_mode() on
function pointers means we no longer need the
tlbmiss_handler_setup_pgd_start symbol so that is removed entirely.

Now that we're declaring our functions as functions GCC stops optimizing
out msk_isa16_mode() & a microMIPS kernel built with either GCC 7.3.0 or
8.1.0 boots successfully.

Signed-off-by: Paul Burton <paul.burton@mips.com>

diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h
index b509371a6b0cce4842ab634c80c658e35d245db6..94414561de0e7b9c46d6d6af17c42de6e8519ab6 100644 (file)
--- a/arch/mips/include/asm/mmu_context.h
+++ b/arch/mips/include/asm/mmu_context.h
@@ -32,6 +32,7 @@ do {                                                                  \
 } while (0)
 
 extern void tlbmiss_handler_setup_pgd(unsigned long);
+extern char tlbmiss_handler_setup_pgd_end[];
 
 /* Note: This is also implemented with uasm in arch/mips/kvm/entry.c */
 #define TLBMISS_HANDLER_SETUP_PGD(pgd)                                 \

diff --git a/arch/mips/include/asm/tlbex.h b/arch/mips/include/asm/tlbex.h
index 4ed7ebed3bf1f060d2f6965618120751e519605a..6d97e23f30abdded461a1b93f17feb9fd5d85a65 100644 (file)
--- a/arch/mips/include/asm/tlbex.h
+++ b/arch/mips/include/asm/tlbex.h
@@ -24,4 +24,13 @@ void build_tlb_write_entry(u32 **p, struct uasm_label **l,
                           struct uasm_reloc **r,
                           enum tlb_write_entry wmode);
 
+extern void handle_tlbl(void);
+extern char handle_tlbl_end[];
+
+extern void handle_tlbs(void);
+extern char handle_tlbs_end[];
+
+extern void handle_tlbm(void);
+extern char handle_tlbm_end[];
+
 #endif /* __ASM_TLBEX_H */

diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 4d9ca9b465ae96235ebc4712ad5dc852c75f6ffb..5e1e580dd1649b28d65f50417def5d99487f3b5a 100644 (file)
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -67,14 +67,12 @@
 #include <asm/mmu_context.h>
 #include <asm/types.h>
 #include <asm/stacktrace.h>
+#include <asm/tlbex.h>
 #include <asm/uasm.h>
 
 extern void check_wait(void);
 extern asmlinkage void rollback_handle_int(void);
 extern asmlinkage void handle_int(void);
-extern u32 handle_tlbl[];
-extern u32 handle_tlbs[];
-extern u32 handle_tlbm[];
 extern asmlinkage void handle_adel(void);
 extern asmlinkage void handle_ades(void);
 extern asmlinkage void handle_ibe(void);

diff --git a/arch/mips/mm/tlb-funcs.S b/arch/mips/mm/tlb-funcs.S
index ae34361b6060c7759fd119424f4906a7f311a1da..00fef578c8cdcfbc1359e476255d81f5c6cd4363 100644 (file)
--- a/arch/mips/mm/tlb-funcs.S
+++ b/arch/mips/mm/tlb-funcs.S
@@ -17,7 +17,6 @@
 
 #define FASTPATH_SIZE  128
 
-EXPORT(tlbmiss_handler_setup_pgd_start)
 LEAF(tlbmiss_handler_setup_pgd)
 1:     j       1b              /* Dummy, will be replaced. */
        .space  64

diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 2223620fb5735a48c488181513ed1d5a29dbf83d..06771429164317c3e8bfe5da8e02e33445fb0b14 100644 (file)
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -31,6 +31,7 @@
 
 #include <asm/cacheflush.h>
 #include <asm/cpu-type.h>
+#include <asm/mmu_context.h>
 #include <asm/pgtable.h>
 #include <asm/war.h>
 #include <asm/uasm.h>
@@ -253,8 +254,10 @@ static void output_pgtable_bits_defines(void)
        pr_debug("\n");
 }
 
-static inline void dump_handler(const char *symbol, const u32 *handler, int count)
+static inline void dump_handler(const char *symbol, const void *start, const void *end)
 {
+       unsigned int count = (end - start) / sizeof(u32);
+       const u32 *handler = start;
        int i;
 
        pr_debug("LEAF(%s)\n", symbol);
@@ -402,12 +405,6 @@ static void build_restore_work_registers(u32 **p)
  * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
  * we cannot do r3000 under these circumstances.
  *
- * Declare pgd_current here instead of including mmu_context.h to avoid type
- * conflicts for tlbmiss_handler_setup_pgd
- */
-extern unsigned long pgd_current[];
-
-/*
  * The R3000 TLB handler is simple.
  */
 static void build_r3000_tlb_refill_handler(void)
@@ -444,8 +441,7 @@ static void build_r3000_tlb_refill_handler(void)
 
        memcpy((void *)ebase, tlb_handler, 0x80);
        local_flush_icache_range(ebase, ebase + 0x80);
-
-       dump_handler("r3000_tlb_refill", (u32 *)ebase, 32);
+       dump_handler("r3000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x80));
 }
 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
@@ -1465,8 +1461,7 @@ static void build_r4000_tlb_refill_handler(void)
 
        memcpy((void *)ebase, final_handler, 0x100);
        local_flush_icache_range(ebase, ebase + 0x100);
-
-       dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
+       dump_handler("r4000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x100));
 }
 
 static void setup_pw(void)
@@ -1568,30 +1563,21 @@ static void build_loongson3_tlb_refill_handler(void)
        uasm_resolve_relocs(relocs, labels);
        memcpy((void *)(ebase + 0x80), tlb_handler, 0x80);
        local_flush_icache_range(ebase + 0x80, ebase + 0x100);
-       dump_handler("loongson3_tlb_refill", (u32 *)(ebase + 0x80), 32);
+       dump_handler("loongson3_tlb_refill",
+                    (u32 *)(ebase + 0x80), (u32 *)(ebase + 0x100));
 }
 
-extern u32 handle_tlbl[], handle_tlbl_end[];
-extern u32 handle_tlbs[], handle_tlbs_end[];
-extern u32 handle_tlbm[], handle_tlbm_end[];
-extern u32 tlbmiss_handler_setup_pgd_start[];
-extern u32 tlbmiss_handler_setup_pgd[];
-extern u32 tlbmiss_handler_setup_pgd_end[];
-
 static void build_setup_pgd(void)
 {
        const int a0 = 4;
        const int __maybe_unused a1 = 5;
        const int __maybe_unused a2 = 6;
-       u32 *p = tlbmiss_handler_setup_pgd_start;
-       const int tlbmiss_handler_setup_pgd_size =
-               tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
+       u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd);
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
        long pgdc = (long)pgd_current;
 #endif
 
-       memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
-                                       sizeof(tlbmiss_handler_setup_pgd[0]));
+       memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
        pgd_reg = allocate_kscratch();
@@ -1644,15 +1630,15 @@ static void build_setup_pgd(void)
        else
                uasm_i_nop(&p);
 #endif
-       if (p >= tlbmiss_handler_setup_pgd_end)
+       if (p >= (u32 *)tlbmiss_handler_setup_pgd_end)
                panic("tlbmiss_handler_setup_pgd space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
-                (unsigned int)(p - tlbmiss_handler_setup_pgd));
+                (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd));
 
        dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
-                                       tlbmiss_handler_setup_pgd_size);
+                                       tlbmiss_handler_setup_pgd_end);
 }
 
 static void
@@ -1921,12 +1907,11 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
 
 static void build_r3000_tlb_load_handler(void)
 {
-       u32 *p = handle_tlbl;
-       const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
+       u32 *p = (u32 *)handle_tlbl;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
 
-       memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
+       memset(p, 0, handle_tlbl_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -1940,24 +1925,23 @@ static void build_r3000_tlb_load_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbl_end)
+       if (p >= (u32 *)handle_tlbl_end)
                panic("TLB load handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbl));
+                (unsigned int)(p - (u32 *)handle_tlbl));
 
-       dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
+       dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_end);
 }
 
 static void build_r3000_tlb_store_handler(void)
 {
-       u32 *p = handle_tlbs;
-       const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
+       u32 *p = (u32 *)handle_tlbs;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
 
-       memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
+       memset(p, 0, handle_tlbs_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -1971,24 +1955,23 @@ static void build_r3000_tlb_store_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbs_end)
+       if (p >= (u32 *)handle_tlbs_end)
                panic("TLB store handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbs));
+                (unsigned int)(p - (u32 *)handle_tlbs));
 
-       dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
+       dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_end);
 }
 
 static void build_r3000_tlb_modify_handler(void)
 {
-       u32 *p = handle_tlbm;
-       const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
+       u32 *p = (u32 *)handle_tlbm;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
 
-       memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
+       memset(p, 0, handle_tlbm_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -2002,14 +1985,14 @@ static void build_r3000_tlb_modify_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbm_end)
+       if (p >= (u32 *)handle_tlbm_end)
                panic("TLB modify handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbm));
+                (unsigned int)(p - (u32 *)handle_tlbm));
 
-       dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
+       dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_end);
 }
 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
@@ -2101,12 +2084,11 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
 static void build_r4000_tlb_load_handler(void)
 {
        u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl);
-       const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
        struct work_registers wr;
 
-       memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
+       memset(p, 0, handle_tlbl_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -2287,25 +2269,24 @@ static void build_r4000_tlb_load_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbl_end)
+       if (p >= (u32 *)handle_tlbl_end)
                panic("TLB load handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbl));
+                (unsigned int)(p - (u32 *)handle_tlbl));
 
-       dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
+       dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_end);
 }
 
 static void build_r4000_tlb_store_handler(void)
 {
        u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs);
-       const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
        struct work_registers wr;
 
-       memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
+       memset(p, 0, handle_tlbs_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -2342,25 +2323,24 @@ static void build_r4000_tlb_store_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbs_end)
+       if (p >= (u32 *)handle_tlbs_end)
                panic("TLB store handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbs));
+                (unsigned int)(p - (u32 *)handle_tlbs));
 
-       dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
+       dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_end);
 }
 
 static void build_r4000_tlb_modify_handler(void)
 {
        u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm);
-       const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
        struct uasm_label *l = labels;
        struct uasm_reloc *r = relocs;
        struct work_registers wr;
 
-       memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
+       memset(p, 0, handle_tlbm_end - (char *)p);
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
@@ -2398,14 +2378,14 @@ static void build_r4000_tlb_modify_handler(void)
        uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
        uasm_i_nop(&p);
 
-       if (p >= handle_tlbm_end)
+       if (p >= (u32 *)handle_tlbm_end)
                panic("TLB modify handler fastpath space exceeded");
 
        uasm_resolve_relocs(relocs, labels);
        pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
-                (unsigned int)(p - handle_tlbm));
+                (unsigned int)(p - (u32 *)handle_tlbm));
 
-       dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
+       dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_end);
 }
 
 static void flush_tlb_handlers(void)