From: Dave Kleikamp Date: Fri, 23 May 2014 21:49:04 +0000 (-0500) Subject: sparc64: new files: kexec_shim.S and machine_kexec.c X-Git-Tag: v4.1.12-92~147^2~3^2~11 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=6fa4477f7e671b4882517a0862d3ee3f65ff4bde;p=users%2Fjedix%2Flinux-maple.git sparc64: new files: kexec_shim.S and machine_kexec.c original patch by Bob Picco Signed-off-by: Dave Kleikamp Cc: Bob Picco --- diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index b8f20ad521c78..acbd76897a17d 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -117,3 +117,6 @@ pc--$(CONFIG_PERF_EVENTS) := perf_event.o obj-$(CONFIG_SPARC64) += $(pc--y) obj-$(CONFIG_SPARC64) += jump_label.o + +obj-$(CONFIG_SPARC64) += kexec_shim.o +obj-$(CONFIG_KEXEC) += machine_kexec.o diff --git a/arch/sparc/kernel/kexec_shim.S b/arch/sparc/kernel/kexec_shim.S new file mode 100644 index 0000000000000..918d470fab0ce --- /dev/null +++ b/arch/sparc/kernel/kexec_shim.S @@ -0,0 +1,319 @@ +#include +#include +#include +#include +#include +#include + .text + .section sparc64_kexec, "axw" +#define KX_SCRATCH ((0x200 << 5) << 1) +#define KX_HV_FAULT_I_TYPE_OFFSET HV_FAULT_I_TYPE_OFFSET +#define KX_HV_FAULT_I_ADDR_OFFSET HV_FAULT_I_ADDR_OFFSET +#define KX_HV_FAULT_D_TYPE_OFFSET HV_FAULT_D_TYPE_OFFSET +#define KX_HV_FAULT_D_ADDR_OFFSET HV_FAULT_D_ADDR_OFFSET +#define KX_NR_NTTE 0x0080 +#define KX_NR_KTTE 0x0088 +#define KX_KTTE 0x0090 +#define KX_NR_ITTE 0x00d0 +#define KX_ITTE 0x00d8 +#define KX_OBP_CIF 0x0118 +#define KX_OBP_SP 0x0120 +#define KX_OBP_TRANSLATIONS 0x0128 +#define BKT(LVL) \ + wrpr %g0, 0, %gl; \ + ldxa [%g0] ASI_SCRATCHPAD, %g1; \ + ldx [%g1 + KX_OBP_CIF], %o4; \ + ldx [%g1 + KX_OBP_SP], %o6; \ + ba,a,pt %xcc, bad_bkt_trap; \ + nop; nop; nop; +#define BKTS(LVL) BKT(LVL) BKT(LVL+1) BKT(LVL+2) BKT(LVL+3) +#define KX_ITLB_MISS \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + KX_HV_FAULT_I_ADDR_OFFSET], %g1; \ + or %g0, %lo(HV_MMU_IMMU), %g3; \ + ba,a,pt %xcc, kx_tlb_miss; \ + nop; nop; nop; nop; +#define KX_DTLB_MISS \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + KX_HV_FAULT_D_ADDR_OFFSET], %g1; \ + or %g0, %lo(HV_MMU_DMMU), %g3; \ + ba,a,pt %xcc, kx_tlb_miss; \ + nop; nop; nop; nop; + .align 0x10000 + .global sparc64_kexec_trap_tl0 +sparc64_kexec_trap_tl0: +tl0_kexec_000: BKT(0x000) BKT(0x001) BKT(0x002) BKT(0x003) +tl0_kexec_004: BKT(0x004) BKT(0x005) BKT(0x006) BKT(0x007) +tl0_kexec_008: BKT(0x008) +tl0_kexec_sun4v_itsb: BKT(0x009) +tl0_kexec_00a: BKT(0x00a) BKT(0x00b) +tl0_kexec_00c: BKT(0x00c) BKT(0x00d) BKT(0x00e) BKT(0x00f) +tl0_kexec_010: BKT(0x010) BKT(0x011) BKT(0x012) BKT(0x013) +tl0_kexec_014: BKT(0x014) BKT(0x015) BKT(0x016) BKT(0x017) +tl0_kexec_018: BKT(0x018) BKT(0x019) BKT(0x01a) BKT(0x01b) +tl0_kexec_01c: BKT(0x01c) BKT(0x01d) BKT(0x01e) BKT(0x01f) +tl0_kexec_020: BKT(0x020) BKT(0x021) BKT(0x022) BKT(0x023) +tl0_kexec_clean_window: + CLEAN_WINDOW +tl0_kexec_028: BKT(0x028) BKT(0x029) BKT(0x02a) BKT(0x02b) +tl0_kexec_02c: BKT(0x02c) BKT(0x02d) BKT(0x02e) BKT(0x02f) +tl0_kexec_030: BKT(0x030) +tl0_kexec_sun4v_dtsb: BKT(0x031) +tl0_kexec_032: BKT(0x032) BKT(0x033) +tl0_kexec_034: BKT(0x034) BKT(0x035) BKT(0x036) BKT(0x037) +tl0_kexec_038: BKT(0x038) BKT(0x039) BKT(0x03a) BKT(0x03b) +tl0_kexec_03c: BKT(0x03c) BKT(0x03d) BKT(0x03e) BKT(0x03f) +tl0_kexec_040: BKT(0x040) BKT(0x041) BKT(0x042) BKT(0x043) +tl0_kexec_044: BKT(0x044) BKT(0x045) BKT(0x046) BKT(0x047) +tl0_kexec_048: BKT(0x048) BKT(0x049) BKT(0x04a) BKT(0x04b) +tl0_kexec_04c: BKT(0x04c) BKT(0x04d) BKT(0x04e) BKT(0x04f) +tl0_kexec_050: BKT(0x050) BKT(0x051) BKT(0x052) BKT(0x053) +tl0_kexec_054: BKT(0x054) BKT(0x055) BKT(0x056) BKT(0x057) +tl0_kexec_058: BKT(0x058) BKT(0x059) BKT(0x05a) BKT(0x05b) +tl0_kexec_05c: BKT(0x05c) BKT(0x05d) BKT(0x05e) BKT(0x05f) +tl0_kexec_060: BKT(0x060) BKT(0x061) BKT(0x062) BKT(0x063) +tl0_kexec_fast_instruction_access_MMU_miss: + KX_ITLB_MISS BKT(0x065) BKT(0x066) BKT(0x067) +tl0_kexec_fast_data_access_MMU_miss: + KX_DTLB_MISS BKT(0x069) BKT(0x06a) BKT(0x06b) +tl0_kexec_06c: BKT(0x06c) BKT(0x06d) BKT(0x06e) BKT(0x06f) +tl0_kexec_070: BKT(0x070) BKT(0x071) BKT(0x072) BKT(0x073) +tl0_kexec_074: BKT(0x074) BKT(0x075) BKT(0x076) BKT(0x077) +tl0_kexec_078: BKT(0x078) BKT(0x079) BKT(0x07a) BKT(0x07b) +tl0_kexec_07c: BKT(0x07c) BKT(0x07d) BKT(0x07e) BKT(0x07f) +tl0_spill_0_normal: + SPILL_0_NORMAL +tl0_kexec_084: BKT(0x084) BKT(0x085) BKT(0x086) BKT(0x087) +tl0_kexec_088: BKT(0x088) BKT(0x089) BKT(0x08a) BKT(0x08b) +tl0_kexec_08c: BKT(0x08c) BKT(0x08d) BKT(0x08e) BKT(0x08f) +tl0_kexec_090: BKT(0x090) BKT(0x091) BKT(0x092) BKT(0x093) +tl0_kexec_094: BKT(0x094) BKT(0x095) BKT(0x096) BKT(0x097) +tl0_kexec_098: BKT(0x098) BKT(0x099) BKT(0x09a) BKT(0x09b) +tl0_kexec_09c: BKT(0x09c) BKT(0x09d) BKT(0x09e) BKT(0x09f) +tl0_kexec_0a0: BKT(0x0a0) BKT(0x0a1) BKT(0x0a2) BKT(0x0a3) +tl0_kexec_0a4: BKT(0x0a4) BKT(0x0a5) BKT(0x0a6) BKT(0x0a7) +tl0_kexec_0a8: BKT(0x0a8) BKT(0x0a9) BKT(0x0aa) BKT(0x0ab) +tl0_kexec_0ac: BKT(0x0ac) BKT(0x0ad) BKT(0x0ae) BKT(0x0af) +tl0_kexec_0b0: BKT(0x0b0) BKT(0x0b1) BKT(0x0b2) BKT(0x0b3) +tl0_kexec_0b4: BKT(0x0b4) BKT(0x0b5) BKT(0x0b6) BKT(0x0b7) +tl0_kexec_0b8: BKT(0x0b8) BKT(0x0b9) BKT(0x0ba) BKT(0x0bb) +tl0_kexec_0bc: BKT(0x0bc) BKT(0x0bd) BKT(0x0be) BKT(0x0bf) +tl0_fill_0_normal: + FILL_0_NORMAL +tl0_kexec_0c4: BKT(0x0c4) BKT(0x0c5) BKT(0x0c6) BKT(0x0c7) +tl0_kexec_0c8: BKT(0x0c8) BKT(0x0c9) BKT(0x0ca) BKT(0x0cb) +tl0_kexec_0cc: BKT(0x0cc) BKT(0x0cd) BKT(0x0ce) BKT(0x0cf) +tl0_kexec_0d0: BKT(0x0d0) BKT(0x0d1) BKT(0x0d2) BKT(0x0d3) +tl0_kexec_0d4: BKT(0x0d4) BKT(0x0d5) BKT(0x0d6) BKT(0x0d7) +tl0_kexec_0d8: BKT(0x0d8) BKT(0x0d9) BKT(0x0da) BKT(0x0db) +tl0_kexec_0dc: BKT(0x0dc) BKT(0x0dd) BKT(0x0de) BKT(0x0df) +tl0_kexec_0e0: BKT(0x0e0) BKT(0x0e1) BKT(0x0e2) BKT(0x0e3) +tl0_kexec_0e4: BKT(0x0e4) BKT(0x0e5) BKT(0x0e6) BKT(0x0e7) +tl0_kexec_0e8: BKT(0x0e8) BKT(0x0e9) BKT(0x0ea) BKT(0x0eb) +tl0_kexec_0ec: BKT(0x0ec) BKT(0x0ed) BKT(0x0ee) BKT(0x0ef) +tl0_kexec_0f0: BKT(0x0f0) BKT(0x0f1) BKT(0x0f2) BKT(0x0f3) +tl0_kexec_0f4: BKT(0x0f4) BKT(0x0f5) BKT(0x0f6) BKT(0x0f7) +tl0_kexec_0f8: BKT(0x0f8) BKT(0x0f9) BKT(0x0fa) BKT(0x0fb) +tl0_kexec_0fc: BKT(0x0fc) BKT(0x0fd) BKT(0x0fe) BKT(0x0ff) +! Note this is a PITA but cleanly done is best. Should my typos be nonexistent. +tl0_kexec_100: BKTS(0x100) BKTS(0x104) BKTS(0x108) BKTS(0x10c) +tl0_kexec_110: BKTS(0x110) BKTS(0x114) BKTS(0x118) BKTS(0x11c) +tl0_kexec_120: BKTS(0x120) BKTS(0x124) BKTS(0x128) BKTS(0x12c) +tl0_kexec_130: BKTS(0x130) BKTS(0x134) BKTS(0x138) BKTS(0x13c) +tl0_kexec_140: BKTS(0x140) BKTS(0x144) BKTS(0x148) BKTS(0x14c) +tl0_kexec_150: BKTS(0x150) BKTS(0x154) BKTS(0x158) BKTS(0x15c) +tl0_kexec_160: BKTS(0x160) BKTS(0x164) BKTS(0x168) BKTS(0x16c) +tl0_kexec_170: BKTS(0x170) BKTS(0x174) BKTS(0x178) BKTS(0x17c) +tl0_kexec_180: BKTS(0x180) BKTS(0x184) BKTS(0x188) BKTS(0x18c) +tl0_kexec_190: BKTS(0x190) BKTS(0x194) BKTS(0x198) BKTS(0x19c) +tl0_kexec_1a0: BKTS(0x1a0) BKTS(0x1a4) BKTS(0x1a8) BKTS(0x1ac) +tl0_kexec_1b0: BKTS(0x1b0) BKTS(0x1b4) BKTS(0x1b8) BKTS(0x1bc) +tl0_kexec_1c0: BKTS(0x1c0) BKTS(0x1c4) BKTS(0x1c8) BKTS(0x1cc) +tl0_kexec_1d0: BKTS(0x1d0) BKTS(0x1d4) BKTS(0x1d8) BKTS(0x1dc) +tl0_kexec_1e0: BKTS(0x1e0) BKTS(0x1e4) BKTS(0x1e8) BKTS(0x1ec) +tl0_kexec_1f0: BKTS(0x1f0) BKTS(0x1f4) BKTS(0x1f8) BKTS(0x1fc) +tl1_kexec_000: BKTS(0x000) BKTS(0x004) BKTS(0x008) BKTS(0x00c) +tl1_kexec_010: BKTS(0x010) BKTS(0x014) BKTS(0x018) BKTS(0x01c) +tl1_kexec_020: BKTS(0x020) BKTS(0x024) BKTS(0x028) BKTS(0x02c) +tl1_kexec_030: BKTS(0x030) BKTS(0x034) BKTS(0x038) BKTS(0x03c) +tl1_kexec_040: BKTS(0x040) BKTS(0x044) BKTS(0x048) BKTS(0x04c) +tl1_kexec_050: BKTS(0x050) BKTS(0x054) BKTS(0x058) BKTS(0x05c) +tl1_kexec_060: BKTS(0x000) +tl1_kexec_fast_instruction_access_MMU_miss: + KX_ITLB_MISS BKT(0x065) BKT(0x066) BKT(0x067) +tl1_kexec_fast_data_access_MMU_miss: + KX_DTLB_MISS BKT(0x069) BKT(0x06a) BKT(0x06b) +tl1_kexec_06c: BKTS(0x06c) +tl1_kexec_070: BKTS(0x070) BKTS(0x074) BKTS(0x078) BKTS(0x07c) +tl1_kexec_080: BKTS(0x080) BKTS(0x084) BKTS(0x088) BKTS(0x08c) +tl1_kexec_090: BKTS(0x090) BKTS(0x094) BKTS(0x098) BKTS(0x09c) +tl1_kexec_0a0: BKTS(0x0a0) BKTS(0x0a4) BKTS(0x0a8) BKTS(0x0ac) +tl1_kexec_0b0: BKTS(0x0b0) BKTS(0x0b4) BKTS(0x0b8) BKTS(0x0bc) +tl1_kexec_0c0: BKTS(0x0c0) BKTS(0x0c4) BKTS(0x0c8) BKTS(0x0cc) +tl1_kexec_0d0: BKTS(0x0d0) BKTS(0x0d4) BKTS(0x0d8) BKTS(0x0dc) +tl1_kexec_0e0: BKTS(0x0e0) BKTS(0x0e4) BKTS(0x0e8) BKTS(0x0ec) +tl1_kexec_0f0: BKTS(0x0f0) BKTS(0x0f4) BKTS(0x0f8) BKTS(0x0fc) +tl1_kexec_100: BKTS(0x100) BKTS(0x104) BKTS(0x108) BKTS(0x10c) +tl1_kexec_110: BKTS(0x110) BKTS(0x114) BKTS(0x118) BKTS(0x11c) +tl1_kexec_120: BKTS(0x120) BKTS(0x124) BKTS(0x128) BKTS(0x12c) +tl1_kexec_130: BKTS(0x130) BKTS(0x134) BKTS(0x138) BKTS(0x13c) +tl1_kexec_140: BKTS(0x140) BKTS(0x144) BKTS(0x148) BKTS(0x14c) +tl1_kexec_150: BKTS(0x150) BKTS(0x154) BKTS(0x158) BKTS(0x15c) +tl1_kexec_160: BKTS(0x160) BKTS(0x164) BKTS(0x168) BKTS(0x16c) +tl1_kexec_170: BKTS(0x170) BKTS(0x174) BKTS(0x178) BKTS(0x17c) +tl1_kexec_180: BKTS(0x180) BKTS(0x184) BKTS(0x188) BKTS(0x18c) +tl1_kexec_190: BKTS(0x190) BKTS(0x194) BKTS(0x198) BKTS(0x19c) +tl1_kexec_1a0: BKTS(0x1a0) BKTS(0x1a4) BKTS(0x1a8) BKTS(0x1ac) +tl1_kexec_1b0: BKTS(0x1b0) BKTS(0x1b4) BKTS(0x1b8) BKTS(0x1bc) +tl1_kexec_1c0: BKTS(0x1c0) BKTS(0x1c4) BKTS(0x1c8) BKTS(0x1cc) +tl1_kexec_1d0: BKTS(0x1d0) BKTS(0x1d4) BKTS(0x1d8) BKTS(0x1dc) +tl1_kexec_1e0: BKTS(0x1e0) BKTS(0x1e4) BKTS(0x1e8) BKTS(0x1ec) +tl1_kexec_1f0: BKTS(0x1f0) BKTS(0x1f4) BKTS(0x1f8) BKTS(0x1fc) +kx_data_start: +kx_hv_fault_status: + .=.+(16<<3) ! HV fault status +kx_nr_tte: ! current tte pinned + .=.+8 +kx_nr_ktte: ! kexec kernel tte + .=.+8 +kx_ktte: + .=.+(8<<3) +kx_nr_itte: ! initrd for kexec-tools --load + .=.+8 +kx_itte: + .=.+(8<<3) +kx_obp_cif: + .=.+8 +kx_obp_sp: + .=.+8 +kx_obp_translations: ! this should suffice + .=.+(24*128) +kx_data_end: +kx_prom_exit: + .asciz "exit" + .align 4 +bad_bkt_trap: + rd %pc, %l0 + or (bad_bkt_trap - kx_prom_exit), %g0, %l1 + sub %l0, %l1, %l1 + ! arguments: "exit", 0 ins, 0 outs + stx %l1, [%sp + 2047 + 128 + 0x00] + stx %g0, [%sp + 2047 + 128 + 0x08] + stx %g0, [%sp + 2047 + 128 + 0x10] + jmpl %o4, %g0 ! goodbye + add %sp, (2047 + 128), %o0 +bad_early: + ldx [%g2 + KX_OBP_SP], %o6 + ldx [%g2 + KX_OBP_CIF], %o4 + ba,a,pt %xcc,bad_bkt_trap + .global kexec_start +kexec_start: + or %o0, %g0, %g1 + or %o1, %g0, %g3 + wrpr %g0, PIL_NORMAL_MAX, %pil + rdpr %pstate, %g2 + andn %g2, PSTATE_IE, %g2 + wrpr %g2, %g0, %pstate + wrpr %g0, 0, %canrestore + wrpr %g0, 0, %otherwin + wrpr %g0, 6, %cansave + wrpr %g0, 6, %cleanwin + wrpr %g0, 0, %cwp + wrpr %g0, 0, %wstate + sethi %hi(KX_SCRATCH), %g2 + add %g1, %g2, %g2 + wrpr %g1, %tba + stxa %g2, [%g0] ASI_SCRATCHPAD + or %g3, %g3, %o0 + mov HV_FAST_MMU_FAULT_AREA_CONF, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, bad_early + + ldx [%g2 + KX_NR_NTTE], %g4 ! unmap current kernel hv pinned tte-s + sethi %hi(REAL_HPAGE_SIZE), %g5 + or %g5, %g5, %g7 + sllx %g4, 22, %g4 + sethi %hi(KERNBASE), %g6 + add %g6, %g4, %g4 +1: or %g6, %g6, %o0 + or %g0, %g0, %o1 + or %g0, %lo(HV_MMU_ALL), %o2 + mov HV_FAST_MMU_UNMAP_PERM_ADDR, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, bad_early ! prom exit for failure + add %g5, %g7, %g5 + subcc %g5, %g4, %g0 + bne 1b + add %g7, %g6, %g6 + + or %g0, %g0, %o0 ! demap all old translations + or %g0, %g0, %o1 + or %g0, %lo(HV_MMU_ALL), %o2 + or %g0, %lo(HV_FAST_MMU_DEMAP_ALL), %o5 + ta HV_FAST_TRAP + + ldx [%g2 + KX_NR_KTTE], %g4 ! map the kexec-ed kernel + sethi %hi(REAL_HPAGE_SIZE), %g5 + sethi %hi(KERNBASE), %g6 + add %g2, %lo(KX_KTTE), %g7 + ldx [%g7 + %g0], %g1 +2: or %g6, %g6, %o0 + or %g0, %g0, %o1 + or %g1, %g1, %o2 + or %g0, %lo(HV_MMU_ALL), %o3 + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, bad_early + add %g7, 8, %g7 + ldx [%g7 + %g0], %g1 + subcc %g4, 1, %g4 + bne 2b + add %g6, %g5, %g6 + + ! Loaded and time to launch. + ! For now leave kernel entry point at 0x404000 + ldx [%g2 + KX_OBP_SP], %o6 + ldx [%g2 + KX_OBP_CIF], %o4 + sethi %hi(KERNBASE), %g1 + sethi %hi(0x4000), %g2 + or %g1, %g2, %g1 + jmpl %g1, %g0 + nop + + ! g1 is VA + ! g2 is HV status area + ! g3 is MMU flag +kx_tlb_miss: + add %g2, %lo(KX_OBP_TRANSLATIONS), %g2 +10: ldx [%g2 + 0x0], %g4 + brz,pn %g4, bad_bkt_trap + nop + ldx [%g2 + 0x8], %g5 + add %g5, %g4, %g5 + cmp %g5, %g1 + bleu %xcc, 20f + cmp %g1, %g5 + bgeu 20f + sub %g1, %g4, %g4 + ba,pt %xcc, 30f + ldx [%g2 + 0x10], %g2 +20: ba,pt %xcc, 10b + add %g2, (3<<3), %g2 +30: add %g2, %g4, %g2 ! TTE + or %o0, %o0, %g7 + or %o1, %o1, %g6 + or %o2, %o2, %g5 + or %o3, %o3, %g4 + or %g1, %g1, %o0 + or %g0, %g0, %o1 + or %g2, %g2, %o2 + or %g3, %g3, %o3 + ta HV_MMU_MAP_ADDR_TRAP + brnz %o0, bad_bkt_trap + or %g7, %g7, %o0 + or %g6, %g6, %o1 + or %g5, %g5, %o2 + or %g4, %g4, %o3 + retry +. = KX_SHIM_SIZE +kexec_end: diff --git a/arch/sparc/kernel/machine_kexec.c b/arch/sparc/kernel/machine_kexec.c new file mode 100644 index 0000000000000..20cca8e5a6d89 --- /dev/null +++ b/arch/sparc/kernel/machine_kexec.c @@ -0,0 +1,431 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kernel.h" + +static atomic_t kexec_strand_wait; +static const unsigned int tte_order = 9U; + +static int kexec_disable_irq; +static int __init setup_kexec_disable_irq(char *str) +{ + kexec_disable_irq = 1; + return 0; +} +__setup("kexec_disable_irq=", setup_kexec_disable_irq); + +static void machine_kexec_disable_irq(void) +{ + struct irq_desc *desc; + unsigned int i; + + if (!kexec_disable_irq) + return; + + for_each_irq_desc(i, desc) { + struct irq_chip *chip; + + chip = irq_desc_get_chip(desc); + if (!chip) + continue; + + if (chip->irq_disable) + chip->irq_disable(&desc->irq_data); + } +} + +static unsigned long kexec_tte(unsigned long paddr) +{ + unsigned long val; + + val = (_PAGE_VALID | _PAGE_SZ64K_4V | + _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V | + _PAGE_EXEC_4V | _PAGE_W_4V); + + return val | paddr; +} + +static unsigned long compute_nr_tte(unsigned long memsz) +{ + return (memsz + ((1UL << (PAGE_SHIFT + tte_order)) - 1)) >> + (PAGE_SHIFT + tte_order); +} + +static unsigned long kexec_nucleus_tte(unsigned long paddr) +{ + unsigned long val; + + val = (_PAGE_VALID | _PAGE_SZ4MB_4V | + _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V | + _PAGE_EXEC_4V | _PAGE_W_4V); + + return val | paddr; +} + +static int kexec_tte_lock(unsigned long tte, unsigned long mmu) +{ + unsigned long rc = sun4v_mmu_map_perm_addr(KEXEC_BASE, 0, tte, mmu); + + if (rc) { + pr_err("kexec_tte_lock: %ld\n", rc); + return -EIO; + } else + return 0; +} + +static unsigned long kexec_map_shim(void) +{ + struct sparc64_kexec_shim *shimp = kexec_shim(); + unsigned long tte_data; + unsigned long phys_addr; + int rc; + + phys_addr = kimage_addr_to_ra(shimp); + tte_data = kexec_tte(phys_addr); + + rc = kexec_tte_lock(tte_data, HV_MMU_DMMU); + if (rc) + goto out; + + rc = kexec_tte_lock(tte_data, HV_MMU_IMMU); + if (rc) + goto out; + + phys_addr = kimage_addr_to_ra(&shimp->hv_fault); +out: + return phys_addr; +} + +int machine_kexec_prepare(struct kimage *image) +{ + struct sparc64_kexec_shim *shimp = kexec_shim(); + int rc = -ENODEV; + + if (tlb_type != hypervisor) { + pr_err("machine_kexec_prepare: kexec is supported only on sun4v.\n"); + goto out; + } else if (image->type == KEXEC_TYPE_CRASH) { + unsigned long nr_tte = compute_nr_tte(image->segment[0].memsz); + + if (crashk_res.start == 0ULL) { + pr_err("machine_kexec_prepare: kexec crash requires crashkernel boot parameter.\n"); + goto out; + } + + if (nr_tte >= NR_KEXEC_TTE) { + pr_err("machine_kexec_prepare:vmlinuz is too large.\n"); + rc = -ENOMEM; + goto out; + } + } + + rc = 0; + shimp->nr_tte = (unsigned long) num_kernel_image_mappings; +out: + return rc; +} + +static void mondo_tear_down(void) +{ + int cpu = smp_processor_id(); + unsigned long hverror; + + hverror = sun4v_cpu_qconf(HV_CPU_QUEUE_CPU_MONDO, 0UL, 0UL) | + sun4v_cpu_qconf(HV_CPU_QUEUE_DEVICE_MONDO, 0UL, 0UL) | + sun4v_cpu_qconf(HV_CPU_QUEUE_RES_ERROR, 0UL, 0UL) | + sun4v_cpu_qconf(HV_CPU_QUEUE_NONRES_ERROR, 0UL, 0UL); + if (hverror) + pr_err("mondo_tead_down failed %ld for cpu %d.\n", + hverror, cpu); +} + +static void stop_strands(void) +{ + unsigned long hverror; + int cpu; + + for_each_online_cpu(cpu) { + if (cpu == smp_processor_id()) + continue; + + hverror = sun4v_cpu_stop(cpu); + if (hverror) + pr_err("machine_crash_shutdown failed with error %ld for cpu= %d.\n", + hverror, cpu); + } +} + +static void machine_capture_other_strands(void *ignore) +{ + crash_save_cpu(get_irq_regs(), smp_processor_id()); + __asm__ __volatile__("flushw"); + if (atomic_read(&nmi_active) > 0) + stop_nmi_watchdog(NULL); + mondo_tear_down(); + atomic_dec(&kexec_strand_wait); +} + +static void destroy_nucleus_tsb(void) +{ + unsigned long hverror = sun4v_mmu_tsb_ctx0(0UL, 0UL); + + if (hverror) + pr_err("destroy_nucleus_tsb: failed with sun4v_mmu_tsb_ctx0 error %ld\n", + hverror); +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + unsigned long msecs; + + if (atomic_read(&nmi_active) > 0) + stop_nmi_watchdog(NULL); + atomic_set(&kexec_strand_wait, num_online_cpus() - 1); + smp_call_function(machine_capture_other_strands, NULL, false); + msecs = 1000; + while ((atomic_read(&kexec_strand_wait) > 0) && msecs) { + mdelay(1); + msecs--; + } + if (atomic_read(&kexec_strand_wait) > 0) + pr_warn("Strand(s) didn't accept IPI for kexec\n"); + + local_irq_disable(); + stop_strands(); + machine_kexec_disable_irq(); + destroy_nucleus_tsb(); + crash_save_cpu(regs, smp_processor_id()); +} + +void machine_kexec_cleanup(struct kimage *image) +{ +} + +static void machine_strand_mondo_rip(void *ignore) +{ + if (atomic_read(&nmi_active) > 0) + stop_nmi_watchdog(NULL); + mondo_tear_down(); + atomic_dec(&kexec_strand_wait); +} + +void machine_shutdown(void) +{ + unsigned long msecs; + + if (atomic_read(&nmi_active) > 0) + stop_nmi_watchdog(NULL); + atomic_set(&kexec_strand_wait, num_online_cpus() - 1); + smp_call_function(machine_strand_mondo_rip, NULL, false); + msecs = 1000; + while ((atomic_read(&kexec_strand_wait) > 0) && msecs) { + mdelay(1); + msecs--; + } + + local_irq_disable(); + stop_strands(); + machine_kexec_disable_irq(); + destroy_nucleus_tsb(); +} + +static int kimage_arch_load_user_data(struct kimage *image, + struct kexec_region *regp, struct kexec_segment *segment, + int (*add_phys_addr)(struct kimage *image, unsigned long page)) +{ + unsigned char __user *buf = segment->buf; + size_t order_size = 1UL << (PAGE_SHIFT + tte_order); + size_t mbytes = segment->bufsz; + unsigned long nr = 0UL; + int rc = -ENOMEM; + struct page *page; + unsigned long p; + int error; + size_t csize; + + while (mbytes) { + page = kimage_alloc_pages(GFP_KERNEL, tte_order); + if (!page) + goto out; + + p = page_to_pfn(page) << PAGE_SHIFT; + + error = add_phys_addr(image, p); + if (error) { + __free_pages(page, tte_order); + goto out; + } + + csize = min_t(size_t, order_size, mbytes); + error = copy_from_user((void *) __va(p), buf, csize); + if (error) + goto out; + + buf = buf + csize; + mbytes = mbytes - csize; + regp->tte[nr] = p; + nr++; + } + rc = 0; + regp->nr_tte = nr; +out: + return rc; +} + +static int kimage_arch_load_normal_segment_initrd(struct kimage *image, + struct kexec_segment *segment, + int (*add_phys_addr)(struct kimage *image, unsigned long page)) +{ + unsigned long nr_tte; + int rc = -ENOMEM; + + nr_tte = compute_nr_tte(segment->memsz); + + if (nr_tte > NR_KEXEC_TTE) { + pr_err("kexec: initrd is too large.\n"); + goto out; + } + + rc = kimage_arch_load_user_data(image, &image->arch.ki_initrd, + segment, add_phys_addr); +out: + return rc; +} + +static int kimage_arch_load_normal_segment_kernel(struct kimage *image, + struct kexec_segment *segment, + int (*add_phys_addr)(struct kimage *image, unsigned long page)) +{ + int i; + unsigned long order_size = 1UL << (PAGE_SHIFT + tte_order); + unsigned long nr_tte; + unsigned int order; + struct page *page; + unsigned long p; + struct kexec_region *regp = &image->arch.ki_kernel; + int rc = -ENOMEM; + + nr_tte = compute_nr_tte(segment->memsz); + + /* For the kernel we need to reserve one pinned TTE for shim. */ + if (nr_tte >= NR_KEXEC_TTE) { + pr_err("kexec: vmlinuz is too large.\n"); + goto out; + } + + /* + * Simple kernel mapping requires contiguous memory + */ + order = tte_order + ilog2(roundup_pow_of_two(nr_tte)); + + page = kimage_alloc_pages(GFP_KERNEL, order); + + if (!page) + goto out; + p = page_to_pfn(page) << PAGE_SHIFT; + + rc = add_phys_addr(image, p); + if (rc) { + __free_pages(page, order); + goto out; + } + + rc = copy_from_user(__va(p), segment->buf, segment->bufsz); + if (rc) + goto out; + + memset((void *)(__va(p) + segment->bufsz), 0, + segment->memsz - segment->bufsz); + + for (i = 0; i < nr_tte; i++) { + regp->tte[i] = kexec_nucleus_tte(p); + p += order_size; + } + regp->nr_tte = nr_tte; +out: + return rc; +} + +int kimage_arch_load_normal_segment(struct kimage *image, + struct kexec_segment *segment, int *arch_status, + int (*add_phys_addr)(struct kimage *image, unsigned long page)) +{ + int rc; + + if (image->arch.ki_kernel.nr_tte) + rc = kimage_arch_load_normal_segment_initrd(image, + segment, add_phys_addr); + else + rc = kimage_arch_load_normal_segment_kernel(image, + segment, add_phys_addr); + + *arch_status = rc; + return 0; +} + +/* All state we require is in the image's arch member. Transfer it to shim. */ +static void machine_shim_load_kexec(struct kimage *image) +{ + struct sparc64_kexec_shim *shimp = kexec_shim(); + + memcpy(&shimp->kernel, &image->arch.ki_kernel, + sizeof(struct kexec_region)); + memcpy(&shimp->initrd, &image->arch.ki_initrd, + sizeof(struct kexec_region)); +} + +/* We need to compute the ttes for the vmlinuz and transfer to shim. */ +static void machine_shim_load_kexec_crash(struct kimage *image) +{ + unsigned long order_size = 1UL << (PAGE_SHIFT + tte_order); + unsigned long tte = kexec_nucleus_tte(image->segment[0].mem); + unsigned long nr_tte = compute_nr_tte(image->segment[0].memsz); + struct sparc64_kexec_shim *shimp = kexec_shim(); + unsigned long nr; + + shimp->initrd.nr_tte = 0UL; + shimp->kernel.nr_tte = nr_tte; + for (nr = 0UL; nr != nr_tte; nr++) { + shimp->kernel.tte[nr] = tte; + tte += order_size; + } +} + +void machine_kexec(struct kimage *image) +{ + unsigned long fault_area; + void (*shim_start)(void *, unsigned long); + void *trap_table = &sparc64_kexec_trap_tl0; + void *kexec_startp = &kexec_start; + unsigned long distance = (unsigned long) (kexec_startp - trap_table); + + mondo_tear_down(); + if (image == kexec_image) + machine_shim_load_kexec(image); + else + machine_shim_load_kexec_crash(image); + shim_start = (void (*)(void *, unsigned long)) (KEXEC_BASE + distance); + fault_area = kexec_map_shim(); + shim_start((void *) KEXEC_BASE, fault_area); + __builtin_unreachable(); +} + +void arch_crash_save_vmcoreinfo(void) +{ +#ifdef CONFIG_NUMA + VMCOREINFO_SYMBOL(node_data); + VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); +#endif +} + +unsigned long paddr_vmcoreinfo_note(void) +{ + return kimage_addr_to_ra(&vmcoreinfo_note); +}