From 17871c8b4be62dd88a9474c85bd076bb6b35b07e Mon Sep 17 00:00:00 2001 From: Allen Pais Date: Tue, 18 Apr 2017 13:40:44 +0530 Subject: [PATCH] sparc64: Remove xl-hugepages and add multi-page size support with several fixes from Bob Picco. Orabug: 25704426 Signed-off-by: Nitin Gupta Signed-off-by: Allen Pais --- arch/sparc/include/asm/hugetlb.h | 6 +- arch/sparc/include/asm/mmu_64.h | 47 +-- arch/sparc/include/asm/mmu_context_64.h | 10 +- arch/sparc/include/asm/page_64.h | 9 +- arch/sparc/include/asm/pgtable_64.h | 14 +- arch/sparc/include/asm/scratchpad.h | 2 +- arch/sparc/include/asm/tlbflush_64.h | 5 +- arch/sparc/include/asm/tsb.h | 14 +- arch/sparc/kernel/sun4v_tlb_miss.S | 71 +--- arch/sparc/kernel/tsb.S | 104 +---- arch/sparc/kernel/vmlinux.lds.S | 15 - arch/sparc/mm/fault_64.c | 69 +-- arch/sparc/mm/hugetlbpage.c | 532 ++++++------------------ arch/sparc/mm/init_64.c | 157 +------ arch/sparc/mm/tlb.c | 34 +- arch/sparc/mm/tsb.c | 216 ++++------ 16 files changed, 336 insertions(+), 969 deletions(-) diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h index 6670f9111d63..c1ac3b0758ab 100644 --- a/arch/sparc/include/asm/hugetlb.h +++ b/arch/sparc/include/asm/hugetlb.h @@ -32,11 +32,9 @@ static inline int is_hugepage_only_range(struct mm_struct *mm, static inline int prepare_hugepage_range(struct file *file, unsigned long addr, unsigned long len) { - struct hstate *h = hstate_file(file); - - if (len & ~huge_page_mask(h)) + if (len & ~HPAGE_MASK) return -EINVAL; - if (addr & ~huge_page_mask(h)) + if (addr & ~HPAGE_MASK) return -EINVAL; return 0; } diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index 93c9a7ebea84..00e6a7a7f1a7 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h @@ -60,11 +60,6 @@ #define CTX_HWBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_HW_MASK) #define CTX_NRBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_NR_MASK) -/* This identifies the three possible tsbs and indices into tsb array. */ -#define MM_TSB_BASE 0 -#define MM_TSB_HUGE 1 -#define MM_TSB_XLHUGE 2 - #ifndef __ASSEMBLY__ #define TSB_ENTRY_ALIGNMENT 16 @@ -87,18 +82,13 @@ struct tsb_config { unsigned long tsb_map_pte; }; +#define MM_TSB_BASE 0 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -/* This is for the tsbs.*/ -#define MM_NUM_TSBS 3 -/* This is the count of huge_pte_count array. */ -#define MM_NUM_HUGEPAGE_SIZES 2 -#define MM_PTES_HUGE 0 -#define MM_PTES_XLHUGE 1 - +#define MM_TSB_HUGE 1 +#define MM_NUM_TSBS 2 #else #define MM_NUM_TSBS 1 -#define MM_NUM_HUGEPAGE_SIZES 0 #endif typedef struct { @@ -107,40 +97,11 @@ typedef struct { struct tsb_config tsb_block[MM_NUM_TSBS]; struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; void *vdso; - unsigned long huge_pte_count[MM_NUM_HUGEPAGE_SIZES]; + unsigned long hugetlb_pte_count; unsigned long thp_pte_count; bool adi; } mm_context_t; -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - -static inline unsigned long xl_hugepage_pte_count(mm_context_t *mm_context) -{ - return mm_context->huge_pte_count[MM_PTES_XLHUGE]; -} - -static inline unsigned long hugepage_pte_count(mm_context_t *mm_context) -{ - return mm_context->huge_pte_count[MM_PTES_HUGE]; -} - -static inline unsigned int hugepage_size_to_pte_count_idx( - unsigned long hugepage_size) -{ - unsigned int pte_count_index = MM_PTES_HUGE; - - if (hugepage_size != HPAGE_SIZE) - pte_count_index = MM_PTES_XLHUGE; - - return pte_count_index; -} - -void __init hv_establish_xl_hugepage_tsb_descriptor(unsigned short pgsz_idx, - unsigned int pgsz_mask); - -#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */ - - #endif /* !__ASSEMBLY__ */ #define TSB_CONFIG_TSB 0x00 diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index cba84144c6df..251bde36d4dc 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -33,7 +33,6 @@ void destroy_context(struct mm_struct *mm); void __tsb_context_switch(unsigned long pgd_pa, struct tsb_config *tsb_base, struct tsb_config *tsb_huge, - struct tsb_config *tsb_xl_huge, unsigned long tsb_descr_pa); static inline void tsb_context_switch(struct mm_struct *mm) @@ -41,14 +40,11 @@ static inline void tsb_context_switch(struct mm_struct *mm) __tsb_context_switch(__pa(mm->pgd), &mm->context.tsb_block[0], #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - (mm->context.tsb_block[MM_TSB_HUGE].tsb ? - &mm->context.tsb_block[MM_TSB_HUGE] : - NULL), - (mm->context.tsb_block[MM_TSB_XLHUGE].tsb ? - &mm->context.tsb_block[MM_TSB_XLHUGE] : + (mm->context.tsb_block[1].tsb ? + &mm->context.tsb_block[1] : NULL) #else - NULL, NULL + NULL #endif , __pa(&mm->context.tsb_descr[0])); } diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index 0992dbb6c127..e98b6500f263 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -17,9 +17,7 @@ #define HPAGE_SHIFT 23 #define REAL_HPAGE_SHIFT 22 -#define XLHPAGE_16GB_SHIFT 34 -#define XLHPAGE_2GB_SHIFT 31 - +#define HPAGE_256MB_SHIFT 28 #define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) @@ -27,16 +25,15 @@ #define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA -#define HUGE_MAX_HSTATE 2 #define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) +#define HUGE_MAX_HSTATE 2 #endif #ifndef __ASSEMBLY__ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) struct pt_regs; -void hugetlb_setup(struct pt_regs *regs, unsigned int tsb_index); -extern unsigned int xl_hugepage_shift; +void hugetlb_setup(struct pt_regs *regs); #endif #define WANT_PAGE_VIRTUAL diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index ceff978dfcfd..c409b44c3c5a 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -404,7 +404,7 @@ static inline unsigned long __pte_default_huge_mask(void) static inline pte_t pte_mkhuge(pte_t pte) { - return pte; + return __pte(pte_val(pte) | __pte_default_huge_mask()); } static inline bool is_default_hugetlb_pte(pte_t pte) @@ -859,7 +859,7 @@ static inline unsigned long __pmd_page(pmd_t pmd) #define pgd_page_vaddr(pgd) \ ((unsigned long) __va(pgd_val(pgd))) #define pgd_present(pgd) (pgd_val(pgd) != 0U) -#define pgd_clear(pgdp) (pgd_val(*(pgd)) = 0UL) +#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL) static inline unsigned long pud_large(pud_t pud) { @@ -908,10 +908,12 @@ static inline unsigned long pud_pfn(pud_t pud) /* Actual page table PTE updates. */ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm); + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift); static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm) + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) { /* It is more efficient to let flush_tlb_kernel_range() * handle init_mm tlb flushes. @@ -920,7 +922,7 @@ static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, * and SUN4V pte layout, so this inline test is fine. */ if (likely(mm != &init_mm) && pte_accessible(mm, orig)) - tlb_batch_add(mm, vaddr, ptep, orig, fullmm); + tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift); } #define __HAVE_ARCH_PMDP_GET_AND_CLEAR @@ -939,7 +941,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t orig = *ptep; *ptep = pte; - maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm); + maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); } #define set_pte_at(mm,addr,ptep,pte) \ diff --git a/arch/sparc/include/asm/scratchpad.h b/arch/sparc/include/asm/scratchpad.h index 2df727e4c102..5e8b01fb3343 100644 --- a/arch/sparc/include/asm/scratchpad.h +++ b/arch/sparc/include/asm/scratchpad.h @@ -8,7 +8,7 @@ #define SCRATCHPAD_UTSBREG1 0x10 #define SCRATCHPAD_UTSBREG2 0x18 /* 0x20 and 0x28, hypervisor only... */ -#define SCRATCHPAD_XLHUGPAGES 0x30 /* Reserved for XL hugepages TSB */ +#define SCRATCHPAD_UNUSED1 0x30 #define SCRATCHPAD_UNUSED2 0x38 /* Reserved for OBP */ #endif /* !(_SPARC64_SCRATCHPAD_H) */ diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h index 8c8e24fedfb8..54be88a6774c 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -8,7 +8,7 @@ #define TLB_BATCH_NR 192 struct tlb_batch { - bool default_huge; + unsigned int hugepage_shift; struct mm_struct *mm; unsigned long tlb_nr; unsigned long active; @@ -17,7 +17,8 @@ struct tlb_batch { void flush_tsb_kernel_range(unsigned long start, unsigned long end); void flush_tsb_user(struct tlb_batch *tb); -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift); /* TLB flush operations. */ diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index ca6a7013bc19..32258e08da03 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h @@ -149,7 +149,7 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * page size in question. So for PMD mappings (which fall on * bit 23, for 8MB per PMD) we must propagate bit 22 for a * 4MB huge page. For huge PUDs (which fall on bit 33, for - * 8GB per PUD), we have to accomodate 256MB and 2GB huge + * 8GB per PUD), we have to accommodate 256MB and 2GB huge * pages. So for those we propagate bits 32 to 28. */ #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \ @@ -199,6 +199,9 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * if it is a HUGE PMD or a normal one. If it is not valid * then jump to FAIL_LABEL. If it is a HUGE PMD, and it * translates to a valid PTE, branch to PTE_LABEL. + * + * We have to propagate the 4MB bit of the virtual address + * because we are fabricating 8MB pages using 4MB hw pages. */ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ @@ -207,11 +210,12 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; sllx REG2, 32, REG2; \ andcc REG1, REG2, %g0; \ be,pt %xcc, 700f; \ - nop; \ + sethi %hi(4 * 1024 * 1024), REG2; \ brgez,pn REG1, FAIL_LABEL; \ - nop; \ - ba PTE_LABEL; \ - nop; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ 700: #else #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S index b5c8edf5233a..6179e19bc9b9 100644 --- a/arch/sparc/kernel/sun4v_tlb_miss.S +++ b/arch/sparc/kernel/sun4v_tlb_miss.S @@ -32,33 +32,17 @@ * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); * tsb_ptr = tsb_base + (tsb_index * 16); */ -#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, \ - PATCH_PROLOGUE, PATCH_EPILOGUE, \ - HASH_SHIFT, TMP1, TMP2) \ +#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ and TSB_PTR, 0x7, TMP1; \ mov 512, TMP2; \ andn TSB_PTR, 0x7, TSB_PTR; \ sllx TMP2, TMP1, TMP2; \ - PATCH_PROLOGUE; \ srlx VADDR, HASH_SHIFT, TMP1; \ - PATCH_EPILOGUE; \ sub TMP2, 1, TMP2; \ and TMP1, TMP2, TMP1; \ sllx TMP1, 4, TMP1; \ add TSB_PTR, TMP1, TSB_PTR; - /* This is for xl_hugepages.*/ -#define PATCH_HASH_SHIFT_PROLOGUE \ - 661:; -#define PATCH_HASH_SHIFT_EPILOGUE \ - .section .sun4v_xl_hugepage_hash_patch, "ax"; \ - .word 661b; \ - .previous; - - /* This is the normal tsb miss case.*/ -#define PATCH_HASH_SHIFT_NOPROLOGUE -#define PATCH_HASH_SHIFT_NOEPILOGUE - sun4v_itlb_miss: /* Load MMU Miss base into %g2. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 @@ -69,8 +53,7 @@ sun4v_itlb_miss: LOAD_ITLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) - COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \ - PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g3, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 @@ -116,8 +99,7 @@ sun4v_dtlb_miss: LOAD_DTLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) - COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \ - PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g3, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 @@ -190,8 +172,7 @@ sun4v_dtsb_miss: /* fallthrough */ sun4v_tsb_miss_common: - COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \ - PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g5, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 @@ -201,8 +182,7 @@ sun4v_tsb_miss_common: cmp %g5, -1 be,pt %xcc, 80f nop - COMPUTE_TSB_PTR(%g5, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \ - PATCH_HASH_SHIFT_NOEPILOGUE, REAL_HPAGE_SHIFT, %g2, %g7) + COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) /* That clobbered %g2, reload it. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 @@ -214,47 +194,6 @@ sun4v_tsb_miss_common: ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 -#ifdef CONFIG_HUGETLB_PAGE - /* - * %g3 -- FAULT_CODE_{D,I}TLB - * %g4 -- virtual address - * %g5 -- pte - * %g6 -- tag - */ - .global sun4v_xl_hugepages -sun4v_xl_hugepages: - sethi %uhi(_PAGE_SPECIAL_4V), %g2 - sllx %g2, 32, %g2 - andcc %g5, %g2, %g0 - be,pt %xcc, 10f - sethi %uhi(_PAGE_VALID | _PAGE_PMD_HUGE | _PAGE_SPECIAL_4V), %g1; - sllx %g1, 32, %g1 - or %g1, _PAGE_SZALL_4V, %g1 - andn %g5, %g1, %g1 - ldxa [%g1 + %g0] ASI_PHYS_USE_EC, %g5 - brgez,pn %g5, tsb_do_fault -10: mov SCRATCHPAD_XLHUGPAGES, %g1 - ldxa [%g1] ASI_SCRATCHPAD, %g1 - cmp %g1, -1 - beq,pn %xcc, 10f - COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_PROLOGUE, \ - PATCH_HASH_SHIFT_EPILOGUE, 0, %g2, %g7) - ba,pt %xcc, tsb_reload - nop -10: SET_GL(1) - rdpr %tl, %g7 - cmp %g7, 1 - bne,pn %xcc, winfix_trampoline - mov %g3, %g4 - ba,pt %xcc, etrap - rd %pc, %g7 - or %g0, MM_TSB_XLHUGE, %o1 - call hugetlb_setup - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop -#endif /* CONFIG_HUGETLB_PAGE */ - sun4v_itlb_error: rdpr %tl, %g1 cmp %g1, 1 diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index d1ebd7563fa4..10689cfd0ad4 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S @@ -29,13 +29,17 @@ */ tsb_miss_dtlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_DMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 tsb_miss_itlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_IMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 /* At this point we have: * %g1 -- PAGE_SIZE TSB entry address @@ -113,57 +117,12 @@ tsb_miss_page_table_walk_sun4v_fastpath: /* Valid PTE is now in %g5. */ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -661: sethi %uhi(_PAGE_SZALL_4U), %g7 + sethi %uhi(_PAGE_PMD_HUGE), %g7 sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZALL_4V, %g7 - nop - .previous - and %g5, %g7, %g2 - - /* This deserves a comment. Should xl_hugepages be selected for - * sun4v the patch order must be for correctness/safety reasons: - * sun4v_xl_hugepage_hash_patch - * sun4v_xl_hugepage_pte_size_patch - * sun4v_xl_hugepage_pte_branch_patch - * . Doing otherwise could result in hangs and thus boot failures. - */ -661: sethi %uhi(_PAGE_SZHUGE_4U), %g7 -662: sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZHUGE_4V, %g7 - nop - .previous - .section .sun4v_xl_hugepage_pte_size_patch, "ax" - .word 662b - subcc %g2, 0, %g0 - .previous - - /* Should it be patched for xl_hugepages, then we need to fix up - * the disp19 target to sun4v_xl_hugepages. - */ -661: + andcc %g5, %g7, %g0 + be,pt %xcc, 60f nop - .section .sun4v_xl_hugepage_pte_branch_patch, "ax" - .word 661b - beq,pn %xcc, 662f -662: - .previous - cmp %g2, %g7 - bne,pt %xcc, 60f - nop - - /* It is the default huge page (8M). We have to propagate - * the 4MB bit of the virtual address because we are - * fabricating 8MB pages using 4MB hw pages. - */ - sethi %hi(4 * 1024 * 1024), %g2 - andn %g5, %g2, %g5 - and %g4, %g2, %g2 - or %g5, %g2, %g5 /* It is a huge page, use huge page TSB entry address we * calculated above. If the huge page TSB has not been @@ -198,14 +157,13 @@ tsb_miss_page_table_walk_sun4v_fastpath: mov %g3, %g4 ba,pt %xcc, etrap rd %pc, %g7 - or %g0, MM_TSB_HUGE, %o1 call hugetlb_setup add %sp, PTREGS_OFF, %o0 ba,pt %xcc, rtrap nop 60: -#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* At this point we have: * %g1 -- TSB entry address @@ -315,6 +273,10 @@ tsb_do_dtlb_fault: nop .previous + /* Clear context ID bits. */ + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline @@ -397,8 +359,7 @@ tsb_flush: * %o0: page table physical address * %o1: TSB base config pointer * %o2: TSB huge config pointer, or NULL if none - * %o3 TSB XL huge config pointer or NULL if none - * %o4: Hypervisor TSB descriptor physical address + * %o3: Hypervisor TSB descriptor physical address * * We have to run this whole thing with interrupts * disabled so that the current cpu doesn't change @@ -416,7 +377,6 @@ __tsb_context_switch: stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] ldx [%o1 + TSB_CONFIG_REG_VAL], %o0 - /* Check hugepage tsb */ brz,pt %o2, 1f mov -1, %g3 @@ -436,34 +396,16 @@ __tsb_context_switch: mov SCRATCHPAD_UTSBREG2, %o5 stxa %g3, [%o5] ASI_SCRATCHPAD - /* Start counting HV tsb descriptors. */ - mov 1, %o0 - cmp %g3, -1 - beq %xcc, 2f - nop - add %o0, 1, %o0 -2: - - /* check xl_hugepage tsb */ - brz,pt %o3, 3f - mov -1, %g3 - ldx [%o3 + TSB_CONFIG_REG_VAL], %g3 -3: - mov SCRATCHPAD_XLHUGPAGES, %o5 - stxa %g3, [%o5] ASI_SCRATCHPAD - + mov 2, %o0 cmp %g3, -1 - beq %xcc, 4f - nop - add %o0, 1, %o0 -4: + move %xcc, 1, %o0 mov HV_FAST_MMU_TSB_CTXNON0, %o5 - mov %o4, %o1 + mov %o3, %o1 ta HV_FAST_TRAP /* Finish up. */ - ba,pt %xcc, 60f + ba,pt %xcc, 9f nop /* SUN4U TSB switch. */ @@ -473,8 +415,8 @@ __tsb_context_switch: stxa %o0, [%o5] ASI_IMMU membar #Sync - ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4 - brz %o4, 60f +2: ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4 + brz %o4, 9f ldx [%o1 + TSB_CONFIG_MAP_PTE], %o5 sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2 @@ -486,7 +428,7 @@ __tsb_context_switch: stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS membar #Sync - brz,pt %o2, 60f + brz,pt %o2, 9f nop ldx [%o2 + TSB_CONFIG_MAP_VADDR], %o4 @@ -498,7 +440,7 @@ __tsb_context_switch: stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS membar #Sync -60: +9: wrpr %g1, %pstate retl diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 580812a66ee2..9d590b886b12 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -123,21 +123,6 @@ SECTIONS *(.swapper_4m_tsb_phys_patch) __swapper_4m_tsb_phys_patch_end = .; } - .sun4v_xl_hugepage_pte_size_patch : { - __sun4v_xl_hugepage_pte_size_patch = .; - *(.sun4v_xl_hugepage_pte_size_patch) - __sun4v_xl_hugepage_pte_size_patch_end = .; - } - .sun4v_xl_hugepage_hash_patch : { - __sun4v_xl_hugepage_hash_patch = .; - *(.sun4v_xl_hugepage_hash_patch) - __sun4v_xl_hugepage_hash_patch_end = .; - } - .sun4v_xl_hugepage_pte_branch_patch : { - __sun4v_xl_hugepage_pte_branch_patch = .; - *(.sun4v_xl_hugepage_pte_branch_patch) - __sun4v_xl_hugepage_pte_branch_patch_end = .; - } .popc_3insn_patch : { __popc_3insn_patch = .; *(.popc_3insn_patch) diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 41583015d6b5..e363428c944c 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -115,15 +115,8 @@ static unsigned int get_user_insn(unsigned long tpc) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if (is_hugetlb_pmd(*pmdp)) { - unsigned long hpage_mask = HPAGE_MASK; - - if (xl_hugepage_shift == XLHPAGE_2GB_SHIFT) - hpage_mask = ~((1UL << xl_hugepage_shift) - 1); - if (pmd_trans_splitting(*pmdp)) - goto out_irq_enable; - pa = pmd_pfn(*pmdp) << PAGE_SHIFT; - pa += tpc & ~hpage_mask; + pa += tpc & ~HPAGE_MASK; /* Use phys bypass so we don't pollute dtlb/dcache. */ __asm__ __volatile__("lduwa [%1] %2, %0" @@ -285,53 +278,10 @@ static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) show_regs(regs); } -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -/* Put this here until there are more consumers.*/ -static void sparc64_hugetlb_tsb_fault(struct pt_regs *regs, - struct mm_struct *mm, - unsigned int hugepage_shift) -{ - unsigned int hugepage_pte_idx, hugepage_idx; - unsigned long mm_rss; - - if (hugepage_shift == xl_hugepage_shift) - hugepage_idx = MM_TSB_XLHUGE; - else - hugepage_idx = MM_TSB_HUGE; - - hugepage_pte_idx = - hugepage_size_to_pte_count_idx(1UL << hugepage_shift); - - mm_rss = mm->context.huge_pte_count[hugepage_pte_idx]; - if (hugepage_idx == MM_TSB_HUGE) { -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss += mm->context.thp_pte_count; -#endif - mm_rss *= REAL_HPAGE_PER_HPAGE; - } - - if (unlikely(mm_rss > - mm->context.tsb_block[hugepage_idx].tsb_rss_limit)) { - if (mm->context.tsb_block[hugepage_idx].tsb) - tsb_grow(mm, hugepage_idx, mm_rss); - else - hugetlb_setup(regs, hugepage_idx); - - } -} -#else -static void sparc64_hugetlb_tsb_fault(struct pt_regs *regs, - struct mm_struct *mm, - unsigned int hugepage_shift) -{ -} -#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */ - asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) { enum ctx_state prev_state = exception_enter(); struct mm_struct *mm = current->mm; - unsigned int hugepage_shift; struct vm_area_struct *vma; unsigned int insn = 0; int si_code, fault_code, fault; @@ -525,10 +475,6 @@ good_area: goto retry; } } - if (is_vm_hugetlb_page(vma)) - hugepage_shift = huge_page_shift(hstate_vma(vma)); - else - hugepage_shift = HPAGE_SHIFT; up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); @@ -538,7 +484,18 @@ good_area: if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); - sparc64_hugetlb_tsb_fault(regs, mm, hugepage_shift); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count; + mm_rss *= REAL_HPAGE_PER_HPAGE; + if (unlikely(mm_rss > + mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) + tsb_grow(mm, MM_TSB_HUGE, mm_rss); + else + hugetlb_setup(regs); + + } +#endif exit_exception: exception_exit(prev_state); return; diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 61bc54992573..6f3851a13418 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -9,24 +9,26 @@ #include #include #include + #include #include +#include #include #include #include #include -#include /* Slightly simplified from the non-hugepage variant because by * definition we don't have to worry about any page coloring stuff */ -static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, + +static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - struct hstate *h = hstate_file(file); + struct hstate *h = hstate_file(filp); unsigned long task_size = TASK_SIZE; struct vm_unmapped_area_info info; @@ -52,13 +54,12 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, } static unsigned long -hugetlb_get_unmapped_area_topdown(struct file *file, - const unsigned long addr0, +hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long len, const unsigned long pgoff, const unsigned long flags) { - struct hstate *h = hstate_file(file); + struct hstate *h = hstate_file(filp); struct mm_struct *mm = current->mm; unsigned long addr = addr0; struct vm_unmapped_area_info info; @@ -91,14 +92,14 @@ hugetlb_get_unmapped_area_topdown(struct file *file, return addr; } -unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags) +unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - unsigned long task_size = TASK_SIZE; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; + unsigned long task_size = TASK_SIZE; if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; @@ -129,168 +130,46 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, pgoff, flags); } -/* Since the hugepage could cover more than one pmd entry and more - * than one pgd entry we must cover all possible conditions. - */ -static pmd_t *huge_pmd_alloc(struct mm_struct *mm, unsigned long addr) -{ - pgd_t *pgd = pgd_offset(mm, addr); - pmd_t *pmd = NULL; - pud_t *pud; - - if (pgd_none(*pgd)) { - pud_t *pud = pud_alloc(mm, pgd, addr); - - if (pud == NULL) - goto out; - } - pud = pud_offset(pgd, addr); - if (pud_none(*pud)) { - pmd = pmd_alloc(mm, pud, addr); - - if (pmd == NULL) - goto out; - } - pmd = pmd_offset(pud, addr); -out: - return pmd; -} - -/* Note, should we fail leave behind the mm state - * which will be cleaned up on exit. - */ -pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, - unsigned long size) -{ - unsigned long start = addr & ~(size - 1); - unsigned long end = start + size; - pte_t *rpte = NULL; - - /* Our caller operates on start's pte which is rpte should we succeed.*/ - for (addr = start; addr < end; addr = addr + PMD_SIZE) { - pmd_t *pmd = huge_pmd_alloc(mm, addr); - pte_t *pte; - - if (!pmd) - goto fail; - if (size != (1UL << XLHPAGE_16GB_SHIFT)) { - rpte = (pte_t *)pmd; - break; - } - - pte = pte_alloc_map(mm, NULL, pmd, addr); - - if (!pte) - goto fail; - else if (!rpte) - rpte = pte; - } - - return rpte; -fail: - return NULL; -} - int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) { return 0; } -/* This function possibly needs to be moved. It will be different - * for sun4u and even possibly for sun4v future cores. Though we have - * no plans to support sun4u at this point. - */ -static unsigned int sun4v_tte_to_shift(pte_t entry) -{ - unsigned long hugepage_tte = pte_val(entry) & _PAGE_SZALL_4V; - unsigned int hugepage_shift; - - switch (hugepage_tte) { - case _PAGE_SZ16GB_4V: - hugepage_shift = XLHPAGE_16GB_SHIFT; - break; - case _PAGE_SZ2GB_4V: - hugepage_shift = XLHPAGE_2GB_SHIFT; - break; - case _PAGE_SZ4MB_4V: - hugepage_shift = REAL_HPAGE_SHIFT; - break; - default: - WARN_ONCE(1, "hugepage_shift: hugepage_tte=0x%lx\n", - hugepage_tte); - hugepage_shift = PAGE_SHIFT; - break; - } - return hugepage_shift; -} - -static unsigned int tte_to_shift(pte_t entry) -{ - unsigned int hugepage_shift; - - if (tlb_type == hypervisor) - hugepage_shift = sun4v_tte_to_shift(entry); - else - hugepage_shift = REAL_HPAGE_SHIFT; - - return hugepage_shift; -} - -static unsigned long tte_to_hugepage_size(pte_t pte) +static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) { - unsigned long hugepage_size = 1UL << tte_to_shift(pte); - - if (hugepage_size == REAL_HPAGE_SIZE) - hugepage_size = HPAGE_SIZE; - return hugepage_size; -} - -static unsigned long tte_to_hugepage_mask(pte_t pte) -{ - unsigned int hugepage_shift = tte_to_shift(pte); - unsigned long hugepage_mask; - - if (hugepage_shift == REAL_HPAGE_SHIFT) - hugepage_shift = HPAGE_SHIFT; - - hugepage_mask = ~((1UL << hugepage_shift) - 1); - - return hugepage_mask; + return entry; } -/* This should also be moved and a noop for sun4u. - * Only include hugepage sizes we plan to support. - */ -static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int hugepage_shift) +static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) { - unsigned long sun4v_hugepage_size = _PAGE_SZ4MB_4V; + unsigned long hugepage_size = _PAGE_SZ4MB_4V; pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; - switch (hugepage_shift) { - /* 16Gb */ - case XLHPAGE_16GB_SHIFT: - sun4v_hugepage_size = _PAGE_SZ16GB_4V; - break; - /* 2Gb */ - case XLHPAGE_2GB_SHIFT: - sun4v_hugepage_size = _PAGE_SZ2GB_4V; + switch (shift) { + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4V; pte_val(entry) |= _PAGE_PMD_HUGE; break; - /* 8Mb */ case HPAGE_SHIFT: pte_val(entry) |= _PAGE_PMD_HUGE; break; default: - WARN_ONCE(hugepage_shift, - "hugepage_shift_to_tte: unsupported " - "hugepage_shift=%u.\n", hugepage_shift); + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); } - pte_val(entry) = pte_val(entry) | sun4v_hugepage_size; + pte_val(entry) = pte_val(entry) | hugepage_size; return entry; } +static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + if (tlb_type == hypervisor) + return sun4v_hugepage_shift_to_tte(entry, shift); + else + return sun4u_hugepage_shift_to_tte(entry, shift); +} + pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, struct page *page, int writeable) { @@ -307,305 +186,158 @@ pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, return hugepage_shift_to_tte(entry, hugepage_shift); } -static void huge_pte_at_flush_update(struct mm_struct *mm, unsigned long addr, - pte_t *pte, pte_t orig, - pte_t *sentinel_pte) -{ - if (pte_val(orig) & _PAGE_VALID) { - if (!(pte_val(*sentinel_pte) & _PAGE_VALID)) { - *sentinel_pte = orig; - tlb_batch_add(mm, addr, pte, orig, false); - } - } -} - -static void form_sentinel(pte_t *sentinel_pte, pte_t entry, pte_t *pte, - unsigned int hugepage_shift) -{ - pte_t sentinel = __pte(_PAGE_VALID | _PAGE_SPECIAL_4V | - (pte_val(entry) & _PAGE_SZALL_4V) | __pa(pte)); - - BUG_ON(__pa(pte) & _PAGE_SZALL_4V); - if (hugepage_shift == 31U) /* 2G page */ - sentinel = __pte(pte_val(sentinel) | _PAGE_PMD_HUGE); - - *sentinel_pte = sentinel; -} - -static bool huge_pte_at_handle_sentinel(pte_t *sentinel_pte, pte_t *pte, - pte_t orig, pte_t entry, - unsigned int hugepage_shift) +static unsigned int sun4v_huge_tte_to_shift(pte_t entry) { - bool rc = true; + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V; + unsigned int shift; - /* Should the original pte be marked valid then - * only update the sentinel. - */ - if (pte_val(orig) & _PAGE_VALID) { - if ((pte_val(orig) & _PAGE_SPECIAL_4V) == 0UL) - *pte = entry; - rc = false; - } else if (pte_val(*sentinel_pte) & _PAGE_VALID) { - *pte = *sentinel_pte; - } else { - form_sentinel(sentinel_pte, entry, pte, hugepage_shift); - *pte = entry; + switch (tte_szbits) { + case _PAGE_SZ256MB_4V: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4V: + shift = REAL_HPAGE_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; } - - return rc; + return shift; } -static bool __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *pte, pte_t entry, pte_t *sentinel_pte, - unsigned int hugepage_shift) +static unsigned int sun4u_huge_tte_to_shift(pte_t entry) { - bool rc = true; - - if (hugepage_shift != REAL_HPAGE_SHIFT) { - pte_t orig = *pte; + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U; + unsigned int shift; - rc = huge_pte_at_handle_sentinel(sentinel_pte, pte, orig, - entry, hugepage_shift); - huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte); - } else - *pte = entry; - - return rc; + switch (tte_szbits) { + case _PAGE_SZ256MB_4U: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4U: + shift = REAL_HPAGE_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; + } + return shift; } -static void __clear_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *pte, pte_t *sentinel_pte, - unsigned int hugepage_shift) +static unsigned int huge_tte_to_shift(pte_t entry) { - if (hugepage_shift != REAL_HPAGE_SHIFT) { - pte_t orig = *pte; + unsigned long shift; - *pte = __pte(0UL); - huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte); - } else - *pte = __pte(0UL); -} - -static bool set_huge_pte_range_at(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr, pte_t *pentry, - pte_t *sentinel_pte, bool set_at) -{ - pte_t *pte = pte_offset_map(pmd, addr); - pte_t *lpte = pte + PTRS_PER_PTE; - pte_t entry = *pentry; - pte_t orig = *(pte_t *)pte; - bool rc = true; - unsigned long orig_addr = addr; - unsigned int hugepage_shift; - - if (set_at) - hugepage_shift = tte_to_shift(entry); + if (tlb_type == hypervisor) + shift = sun4v_huge_tte_to_shift(entry); else - hugepage_shift = tte_to_shift(*pte); - - for (; pte < lpte; pte++, addr = addr + PAGE_SIZE) { - if (set_at) { - rc = __set_huge_pte_at(mm, addr, pte, entry, - sentinel_pte, hugepage_shift); - if (!rc) - break; - pte_val(entry) = pte_val(entry) + PAGE_SIZE; - } else - __clear_huge_pte_at(mm, addr, pte, sentinel_pte, - hugepage_shift); - } - if (set_at) - *pentry = entry; - - if (hugepage_shift == REAL_HPAGE_SHIFT) { - /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - maybe_tlb_batch_add(mm, orig_addr, pte, orig, 0); - maybe_tlb_batch_add(mm, orig_addr + REAL_HPAGE_SIZE, - pte, orig, 0); - } + shift = sun4u_huge_tte_to_shift(entry); - return rc; + if (shift == PAGE_SHIFT) + WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n", + pte_val(entry)); + + return shift; } -static bool __set_huge_pmd_at(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr, unsigned long end, pte_t *pentry, - pte_t *sentinel_pte, bool set_at) +static unsigned long huge_tte_to_size(pte_t pte) { - bool rc; - pte_t orig; - pte_t entry; - unsigned long next; - unsigned long hugepage_shift; - - rc = true; - orig = *(pte_t *)pmd; - entry = *pentry; - - if (set_at) { - hugepage_shift = tte_to_shift(entry); - if (hugepage_shift == REAL_HPAGE_SHIFT) { - *pmd = __pmd(pte_val(entry)); - } else { - do { - next = pmd_addr_end(addr, end); - rc = __set_huge_pte_at(mm, addr, (pte_t *)pmd, - entry, sentinel_pte, hugepage_shift); - if (!rc) - break; - } while (pmd++, addr = next, addr != end); - } - *pentry = entry; - } else { - hugepage_shift = tte_to_shift(orig); - if (hugepage_shift == REAL_HPAGE_SHIFT) { - *pmd = __pmd(0); - } else { - do { - next = pmd_addr_end(addr, end); - __clear_huge_pte_at(mm, addr, (pte_t *)pmd, - sentinel_pte, hugepage_shift); - } while (pmd++, addr = next, addr != end); - } - } - - if (hugepage_shift == REAL_HPAGE_SHIFT) { - /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - maybe_tlb_batch_add(mm, addr, (pte_t *)pmd, orig, 0); - maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, - (pte_t *)pmd, orig, 0); - } + unsigned long size = 1UL << huge_tte_to_shift(pte); - return rc; + if (size == REAL_HPAGE_SIZE) + size = HPAGE_SIZE; + return size; } -static bool set_huge_pmd_at(struct mm_struct *mm, pud_t *pud, - unsigned long addr, unsigned long end, - pte_t *pentry, pte_t *sentinel_pte, bool set_at) +pte_t *huge_pte_alloc(struct mm_struct *mm, + unsigned long addr, unsigned long sz) { - pmd_t *pmd = pmd_offset(pud, addr); - unsigned long next; - bool rc; - unsigned int is_huge_pmd; - - if (set_at) - is_huge_pmd = is_hugetlb_pmd(__pmd(pte_val(*pentry))); - else - is_huge_pmd = is_hugetlb_pmd(*pmd); + pgd_t *pgd; + pud_t *pud; + pte_t *pte = NULL; - if (is_huge_pmd) - return __set_huge_pmd_at(mm, pmd, addr, end, pentry, - sentinel_pte, set_at); + pgd = pgd_offset(mm, addr); + pud = pud_alloc(mm, pgd, addr); + if (pud) + pte = (pte_t *)pmd_alloc(mm, pud, addr); - do { - next = pmd_addr_end(addr, end); - rc = set_huge_pte_range_at(mm, pmd, addr, pentry, - sentinel_pte, set_at); - } while (pmd++, addr = next, ((addr != end) && rc)); - return rc; + return pte; } -static bool set_huge_pud_at(struct mm_struct *mm, pgd_t *pgd, - unsigned long addr, unsigned long end, - pte_t *pentry, pte_t *sentinel_pte, bool set_at) +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) { - pud_t *pud = pud_offset(pgd, addr); - unsigned long next; - bool rc; + pgd_t *pgd; + pud_t *pud; + pte_t *pte = NULL; - do { - next = pud_addr_end(addr, end); - rc = set_huge_pmd_at(mm, pud, addr, next, pentry, - sentinel_pte, set_at); - } while (pud++, addr = next, ((addr != end) && rc)); - return rc; + pgd = pgd_offset(mm, addr); + if (!pgd_none(*pgd)) { + pud = pud_offset(pgd, addr); + if (!pud_none(*pud)) + pte = (pte_t *)pmd_offset(pud, addr); + } + return pte; } -/* entry must be the first pte of the hugepage. Otherwise entry - * must be adjusted before we enter the loop for set_pte_at and - * aligned physically to match the hugepage_size. This is equally - * true of other locations where HUGETLB_PAGE_ORDER is used within - * this module for mainline as of 7/20/2014. - */ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { - pte_t sentinel_pte = __pte(0UL); - unsigned long hugepage_size = tte_to_hugepage_size(entry); - unsigned long hugepage_mask = tte_to_hugepage_mask(entry); - unsigned long start = addr & hugepage_mask; - unsigned long end = start + hugepage_size; - pgd_t *pgd = pgd_offset(mm, start); - unsigned long next; - bool rc; + unsigned int i, nptes, hugepage_shift; + unsigned long size; + pte_t orig; - if (!pte_present(*ptep) && pte_present(entry)) { - unsigned int pte_count_idx = - hugepage_size_to_pte_count_idx(hugepage_size); + size = huge_tte_to_size(entry); + nptes = size >> PMD_SHIFT; - mm->context.huge_pte_count[pte_count_idx]++; - } + if (!pte_present(*ptep) && pte_present(entry)) + mm->context.hugetlb_pte_count += nptes; - do { - next = pgd_addr_end(start, end); - rc = set_huge_pud_at(mm, pgd, start, next, &entry, - &sentinel_pte, true); - } while (pgd++, start = next, ((start != end) && rc)); + addr &= ~(size - 1); + orig = *ptep; + hugepage_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig); + + for (i = 0; i < nptes; i++) + ptep[i] = __pte(pte_val(entry) + (i << PMD_SHIFT)); + + maybe_tlb_batch_add(mm, addr, ptep, orig, 0, hugepage_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0, + hugepage_shift); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_t sentinel_pte = __pte(0UL); - pte_t entry = *ptep; - unsigned long hugepage_size = tte_to_hugepage_size(entry); - unsigned long hugepage_mask = tte_to_hugepage_mask(entry); - unsigned long start = addr & hugepage_mask; - unsigned long end = start + hugepage_size; - pgd_t *pgd = pgd_offset(mm, start); - unsigned long next; - bool rc; - - if (pte_present(entry)) { - unsigned int pte_count_idx = - hugepage_size_to_pte_count_idx(hugepage_size); - - mm->context.huge_pte_count[pte_count_idx]--; - } + unsigned int i, nptes, hugepage_shift; + unsigned long size; + pte_t entry; - do { - next = pgd_addr_end(start, end); - rc = set_huge_pud_at(mm, pgd, start, next, &entry, - &sentinel_pte, false); - } while (pgd++, start = next, ((start != end) && rc)); + entry = *ptep; + size = huge_tte_to_size(entry); + nptes = size >> PMD_SHIFT; + hugepage_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry); - return entry; -} + if (pte_present(entry)) + mm->context.hugetlb_pte_count -= nptes; -pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) -{ - pte_t *pte = NULL; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; + addr &= ~(size - 1); + for (i = 0; i < nptes; i++) + ptep[i] = __pte(0UL); - pgd = pgd_offset(mm, addr); - if (!pgd_none(*pgd)) { - pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (xl_hugepage_shift != XLHPAGE_16GB_SHIFT) - pte = (pte_t *)pmd; - else if (!pmd_none(*pmd)) - pte = pte_offset_map(pmd, addr); - } - } + maybe_tlb_batch_add(mm, addr, ptep, entry, 0, hugepage_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, + hugepage_shift); - return pte; + return entry; } int pmd_huge(pmd_t pmd) { - return 0; + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID; } int pud_huge(pud_t pud) diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 11c31c9176f4..0780d4178a51 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -441,145 +441,38 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde } #ifdef CONFIG_HUGETLB_PAGE -unsigned int xl_hugepage_shift; -static unsigned long xl_hugepage_pte; - -static bool is_xl_hugetlb_pte(pte_t pte) -{ - bool rc = false; - - if (!xl_hugepage_pte) - goto out; - else if ((pte_val(pte) & _PAGE_SZALL_4V) == xl_hugepage_pte) - rc = true; -out: - return rc; -} - -static void __init sun4v_xl_hugepage_hash_patch(void) -{ - extern unsigned int __sun4v_xl_hugepage_hash_patch; - unsigned *insn, *p; - - p = &__sun4v_xl_hugepage_hash_patch; - insn = (unsigned int *)(unsigned long)*p; - *insn = *insn | xl_hugepage_shift; - __asm__ __volatile__("flush %0\n\t" - : /* no outputs */ - : "r" (insn)); -} - -static void __init sun4v_xl_hugepage_pte_size_patch(void) -{ - extern unsigned int __sun4v_xl_hugepage_pte_size_patch; - unsigned *insn, *p; - - p = &__sun4v_xl_hugepage_pte_size_patch; - insn = (unsigned int *)(unsigned long)*p; - p++; - /* It is a simm13 in subcc instruction.*/ - *insn = *p | (unsigned int) xl_hugepage_pte; - __asm__ __volatile__("flush %0\n\t" - : /* no outputs */ - : "r" (insn)); -} - -static void __init sun4v_xl_hugepage_pte_branch_patch(void) -{ - extern unsigned int __sun4v_xl_hugepage_pte_branch_patch; - extern unsigned int sun4v_xl_hugepages; - unsigned int btarget = (unsigned int) - (unsigned long) &sun4v_xl_hugepages; - unsigned int *insn, *p, disp19; - - p = &__sun4v_xl_hugepage_pte_branch_patch; - insn = (unsigned int *)(unsigned long)*p; - /* Instruction which needs to be a bne,pt to sun4v_xl_hugepages.*/ - p++; - disp19 = (btarget - (unsigned int) (unsigned long) insn); - disp19 = disp19 >> 2; - disp19 = disp19 & (0x7ffff); - *insn = *p & ~0x7ffff; - *insn = *insn | disp19; - __asm__ __volatile__("flush %0\n\t" - : /* no outputs */ - : "r" (insn)); -} - -static int __init setup_xl_hugepage(unsigned int hugepage_shift) +static int __init setup_hugepagesz(char *string) { + unsigned long long hugepage_size; + unsigned int hugepage_shift; unsigned short hv_pgsz_idx; unsigned int hv_pgsz_mask; int rc = 0; + hugepage_size = memparse(string, &string); + hugepage_shift = ilog2(hugepage_size); + switch (hugepage_shift) { - case XLHPAGE_16GB_SHIFT: - xl_hugepage_pte = _PAGE_SZ16GB_4V; - hv_pgsz_mask = HV_PGSZ_MASK_16GB; - hv_pgsz_idx = HV_PGSZ_IDX_16GB; + case HPAGE_256MB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_256MB; + hv_pgsz_idx = HV_PGSZ_IDX_256MB; break; - case XLHPAGE_2GB_SHIFT: - xl_hugepage_pte = _PAGE_SZ2GB_4V; - hv_pgsz_mask = HV_PGSZ_MASK_2GB; - hv_pgsz_idx = HV_PGSZ_IDX_2GB; + case HPAGE_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_4MB; + hv_pgsz_idx = HV_PGSZ_IDX_4MB; break; default: - goto out; + hv_pgsz_mask = 0; } if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) { - pr_warn("hugepagesz=%luM not supported by MMU.\n", - (1UL << (hugepage_shift - 20))); + pr_warn("hugepagesz=%llu not supported by MMU.\n", + hugepage_size); goto out; } - if (xl_hugepage_shift) { - pr_warn("hugepagesz=%luM in use and hugepagesz=%luM" - " ignored.\n", - (1UL << (xl_hugepage_shift - 20)), - (1UL << (hugepage_shift - 20))); - goto out; - } - - xl_hugepage_shift = hugepage_shift; - sun4v_xl_hugepage_hash_patch(); - sun4v_xl_hugepage_pte_size_patch(); - sun4v_xl_hugepage_pte_branch_patch(); - hv_establish_xl_hugepage_tsb_descriptor(hv_pgsz_idx, hv_pgsz_mask); - hugetlb_add_hstate(xl_hugepage_shift - PAGE_SHIFT); + hugetlb_add_hstate(hugepage_shift - PAGE_SHIFT); rc = 1; -out: - return rc; -} - -static int __init setup_hugepagesz(char *string) -{ - unsigned long long xl_hugepage_size; - int rc = 0; - - if (tlb_type != hypervisor) - goto out; - - xl_hugepage_size = memparse(string, &string); - - switch (xl_hugepage_size) { - case 1UL << XLHPAGE_16GB_SHIFT: - case 1UL << XLHPAGE_2GB_SHIFT: - { - unsigned int hugepage_shift = ilog2(xl_hugepage_size); - - rc = setup_xl_hugepage(hugepage_shift); - break; - } - default: - pr_warn("hugepage size(0x%llx) not supported.\n", - xl_hugepage_size); - break; - case HPAGE_SIZE: - hugetlb_add_hstate(HUGETLB_PAGE_ORDER); - rc = 1; - break; - } out: return rc; @@ -609,17 +502,13 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * spin_lock_irqsave(&mm->context.lock, flags); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if ((mm->context.huge_pte_count[MM_PTES_HUGE] || - mm->context.thp_pte_count) && is_default_hugetlb_pte(pte)) { - /* We are fabricating 8MB pages using 4MB real hw pages */ + if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) && + is_hugetlb_pmd(__pmd(pte_val(pte)))) { + /* We are fabricating 8MB pages using 4MB real hw pages. */ pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, address, pte_val(pte)); - } else if (mm->context.huge_pte_count[MM_PTES_XLHUGE] && - is_xl_hugetlb_pte(pte)) - __update_mmu_tsb_insert(mm, MM_TSB_XLHUGE, xl_hugepage_shift, - address, pte_val(pte)); - else + } else #endif __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, address, pte_val(pte)); @@ -3319,7 +3208,7 @@ static void context_reload(void *__data) load_secondary_context(mm); } -void hugetlb_setup(struct pt_regs *regs, unsigned int tsb_index) +void hugetlb_setup(struct pt_regs *regs) { struct mm_struct *mm = current->mm; struct tsb_config *tp; @@ -3337,9 +3226,9 @@ void hugetlb_setup(struct pt_regs *regs, unsigned int tsb_index) die_if_kernel("HugeTSB in atomic", regs); } - tp = &mm->context.tsb_block[tsb_index]; + tp = &mm->context.tsb_block[MM_TSB_HUGE]; if (likely(tp->tsb == NULL)) - tsb_grow(mm, tsb_index, 0); + tsb_grow(mm, MM_TSB_HUGE, 0); tsb_context_switch(mm); smp_tsb_sync(mm); diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index 91ab380fbc50..1aa52bdeea36 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void) } static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, - bool exec, bool default_huge) + bool exec, unsigned int hugepage_shift) { struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; @@ -84,19 +84,19 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, } if (!tb->active) { - flush_tsb_user_page(mm, vaddr, default_huge); + flush_tsb_user_page(mm, vaddr, hugepage_shift); global_flush_tlb_page(mm, vaddr); goto out; } if (nr == 0) { tb->mm = mm; - tb->default_huge = default_huge; + tb->hugepage_shift = hugepage_shift; } - if (tb->default_huge != default_huge) { + if (tb->hugepage_shift != hugepage_shift) { flush_tlb_pending(); - tb->default_huge = default_huge; + tb->hugepage_shift = hugepage_shift; nr = 0; } @@ -110,10 +110,9 @@ out: } void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm) + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) { - bool default_huge = is_default_hugetlb_pte(orig); - if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); @@ -139,7 +138,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, no_cache_flush: if (!fullmm) - tlb_batch_add_one(mm, vaddr, pte_exec(orig), default_huge); + tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -155,7 +154,7 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, if (pte_val(*pte) & _PAGE_VALID) { bool exec = pte_exec(*pte); - tlb_batch_add_one(mm, vaddr, exec, false); + tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT); } pte++; vaddr += PAGE_SIZE; @@ -180,16 +179,16 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, * for huge zero page. Huge zero pages are like hugetlb * pages in that there is no RSS, but there is the need * for TSB entries. So, huge zero page counts go into - * huge_pte_count[MM_PTES_HUGE]. + * hugetlb_pte_count. */ if (pmd_val(pmd) & _PAGE_PMD_HUGE) { if (is_huge_zero_page(pmd_page(pmd))) - mm->context.huge_pte_count[MM_PTES_HUGE]++; + mm->context.hugetlb_pte_count++; else mm->context.thp_pte_count++; } else { if (is_huge_zero_page(pmd_page(orig))) - mm->context.huge_pte_count[MM_PTES_HUGE]--; + mm->context.hugetlb_pte_count--; else mm->context.thp_pte_count--; } @@ -210,9 +209,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pte_t orig_pte = __pte(pmd_val(orig)); bool exec = pte_exec(orig_pte); - tlb_batch_add_one(mm, addr, exec, true); + tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT); tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, - true); + REAL_HPAGE_SHIFT); } else { tlb_batch_pmd_scan(mm, addr, orig); } @@ -233,14 +232,13 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); /* - * set_pmd_at() will not be called in a way to decrement the - * context.thp_pte_count when splitting a THP, so do it now. + * set_pmd_at() will not be called in a way to decrement + * thp_pte_count when splitting a THP, so do it now. * Sanity check pmd before doing the actual decrement. */ if ((pmd_val(entry) & _PAGE_PMD_HUGE) && !is_huge_zero_page(pmd_page(entry))) (vma->vm_mm)->context.thp_pte_count--; - } void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 766516237065..f38df0278ab4 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -87,6 +87,33 @@ static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned int hpage_entries; + unsigned int i; + + hpage_entries = 1 << (hugepage_shift - hash_shift); + for (i = 0; i < hpage_entries; i++) + __flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift, + nentries); +} + +static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned long i; + + for (i = 0; i < tb->tlb_nr; i++) + __flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, + nentries, hugepage_shift); +} +#endif + void flush_tsb_user(struct tlb_batch *tb) { struct mm_struct *mm = tb->mm; @@ -94,7 +121,7 @@ void flush_tsb_user(struct tlb_batch *tb) spin_lock_irqsave(&mm->context.lock, flags); - if (!tb->default_huge) { + if (tb->hugepage_shift == PAGE_SHIFT) { base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) @@ -102,33 +129,26 @@ void flush_tsb_user(struct tlb_batch *tb) __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (tb->default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); + __flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries, + tb->hugepage_shift); } #endif -#ifdef CONFIG_HUGETLB_PAGE - if (!tb->default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) { - base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb; - nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries; - base = __pa(base); - __flush_tsb_one(tb, xl_hugepage_shift, base, nentries); - } -#endif /* CONFIG_HUGETLB_PAGE */ spin_unlock_irqrestore(&mm->context.lock, flags); } void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, - bool default_huge) + unsigned int hugepage_shift) { unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); - if (!default_huge) { + if (hugepage_shift == PAGE_SHIFT) { base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) @@ -136,22 +156,15 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); + __flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, + nentries, hugepage_shift); } #endif -#ifdef CONFIG_HUGETLB_PAGE - if (!default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) { - base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb; - nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries; - base = __pa(base); - __flush_tsb_one_entry(base, vaddr, xl_hugepage_shift, nentries); - } -#endif /* CONFIG_HUGETLB_PAGE */ spin_unlock_irqrestore(&mm->context.lock, flags); } @@ -161,56 +174,9 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB -static unsigned short hv_pgsz_idx_xlhuge; -static unsigned int hv_pgsz_mask_xlhuge; - -void __init hv_establish_xl_hugepage_tsb_descriptor(unsigned short pgsz_idx, - unsigned int pgsz_mask) -{ - hv_pgsz_idx_xlhuge = pgsz_idx; - hv_pgsz_mask_xlhuge = pgsz_mask; -} #endif -static void sun4v_fill_tsb_descriptor(struct mm_struct *mm) -{ - struct hv_tsb_descr *htd = &mm->context.tsb_descr[0]; - unsigned int tsb_idx; - - for (tsb_idx = 0U; tsb_idx < MM_NUM_TSBS; tsb_idx++) { - /* Should there not be a tsb then skip it.*/ - if (!mm->context.tsb_block[tsb_idx].tsb) - continue; - - switch (tsb_idx) { - case MM_TSB_BASE: - htd->pgsz_mask = HV_PGSZ_MASK_BASE; - htd->pgsz_idx = HV_PGSZ_IDX_BASE; - break; -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - case MM_TSB_HUGE: - htd->pgsz_mask = HV_PGSZ_MASK_HUGE; - htd->pgsz_idx = HV_PGSZ_IDX_HUGE; - break; - case MM_TSB_XLHUGE: - htd->pgsz_mask = hv_pgsz_mask_xlhuge; - htd->pgsz_idx = hv_pgsz_idx_xlhuge; - break; -#endif - default: - BUG(); - } - htd->assoc = 1; - htd->num_ttes = mm->context.tsb_block[tsb_idx].tsb_nentries; - htd->ctx_idx = 0; - htd->tsb_base = __pa(mm->context.tsb_block[tsb_idx].tsb); - htd->resv = 0; - htd++; - } -} - -static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, - unsigned long tsb_bytes) +static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) { unsigned long tsb_reg, base, tsb_paddr; unsigned long page_sz, tte; @@ -224,7 +190,6 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, break; #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) case MM_TSB_HUGE: - case MM_TSB_XLHUGE: base = TSBMAP_4M_BASE; break; #endif @@ -307,8 +272,40 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, mm->context.tsb_block[tsb_idx].tsb_map_pte = tte; } - if (tlb_type == hypervisor) - sun4v_fill_tsb_descriptor(mm); + /* Setup the Hypervisor TSB descriptor. */ + if (tlb_type == hypervisor) { + struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx]; + + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_idx = HV_PGSZ_IDX_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + hp->pgsz_idx = HV_PGSZ_IDX_HUGE; + break; +#endif + default: + BUG(); + } + hp->assoc = 1; + hp->num_ttes = tsb_bytes / 16; + hp->ctx_idx = 0; + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_mask = HV_PGSZ_MASK_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + hp->pgsz_mask = HV_PGSZ_MASK_HUGE; + break; +#endif + default: + BUG(); + } + hp->tsb_base = tsb_paddr; + hp->resv = 0; + } } struct kmem_cache *pgtable_cache __read_mostly; @@ -518,55 +515,15 @@ retry_tsb_alloc: } } -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -static void capture_and_clear_huge_pte_counts(mm_context_t *mm_context, - unsigned long *thp_pte_count, - unsigned long *capture_array) -{ - unsigned int hugepage_idx; - - *thp_pte_count = mm_context->thp_pte_count; - mm_context->thp_pte_count = 0UL; - - for (hugepage_idx = 0UL; hugepage_idx != MM_NUM_HUGEPAGE_SIZES; - hugepage_idx++) { - capture_array[hugepage_idx] = - mm_context->huge_pte_count[hugepage_idx]; - mm_context->huge_pte_count[hugepage_idx] = 0UL; - } -} - -static void -captured_hugepage_pte_count_grow_tsb(struct mm_struct *mm, - unsigned long *thp_pte_count, - unsigned long *capture_huge_pte_count) -{ - if (unlikely(capture_huge_pte_count[MM_PTES_HUGE])) - if (unlikely(capture_huge_pte_count[MM_PTES_HUGE]) || *thp_pte_count) - tsb_grow(mm, MM_TSB_HUGE, - (capture_huge_pte_count[MM_PTES_HUGE] + - *thp_pte_count) * REAL_HPAGE_PER_HPAGE); - - if (unlikely(capture_huge_pte_count[MM_PTES_XLHUGE])) - tsb_grow(mm, MM_TSB_XLHUGE, - capture_huge_pte_count[MM_PTES_XLHUGE]); -} -#else -static void capture_and_clear_huge_pte_counts(mm_context_t *mm_context, - unsigned long *thp_pte_count, - unsigned long *capture_array) {} -static void -captured_hugepage_pte_count_grow_tsb(struct mm_struct *mm, - unsigned long *thp_pte_count, - unsigned long *capture_huge_pte_count) {} -#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */ - static atomic_t nctxs = ATOMIC_INIT(0); int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { - unsigned long capture_huge_pte_count[MM_NUM_HUGEPAGE_SIZES]; + unsigned long mm_rss = get_mm_rss(mm); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + unsigned long saved_hugetlb_pte_count; unsigned long saved_thp_pte_count; +#endif unsigned int i; int max_nctx = max_user_nctx; int ret = 0; @@ -591,12 +548,18 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; - /* We reset it to zero because the fork() page copying +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + /* We reset them to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. */ - capture_and_clear_huge_pte_counts(&mm->context, &saved_thp_pte_count, - capture_huge_pte_count); + saved_hugetlb_pte_count = mm->context.hugetlb_pte_count; + saved_thp_pte_count = mm->context.thp_pte_count; + mm->context.hugetlb_pte_count = 0; + mm->context.thp_pte_count = 0; + + mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE); +#endif /* copy_mm() copies over the parent's mm_struct before calling * us, so we need to zero out the TSB pointer or else tsb_grow() @@ -608,11 +571,14 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) /* If this is fork, inherit the parent's TSB size. We would * grow it to that size on the first page fault anyways. */ - tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm) - - saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); + tsb_grow(mm, MM_TSB_BASE, mm_rss); - captured_hugepage_pte_count_grow_tsb(mm, &saved_thp_pte_count, - capture_huge_pte_count); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, + (saved_hugetlb_pte_count + saved_thp_pte_count) * + REAL_HPAGE_PER_HPAGE); +#endif if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) { ret = -ENOMEM; -- 2.50.1