static inline int prepare_hugepage_range(struct file *file,
unsigned long addr, unsigned long len)
{
- struct hstate *h = hstate_file(file);
-
- if (len & ~huge_page_mask(h))
+ if (len & ~HPAGE_MASK)
return -EINVAL;
- if (addr & ~huge_page_mask(h))
+ if (addr & ~HPAGE_MASK)
return -EINVAL;
return 0;
}
#define CTX_HWBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_HW_MASK)
#define CTX_NRBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_NR_MASK)
-/* This identifies the three possible tsbs and indices into tsb array. */
-#define MM_TSB_BASE 0
-#define MM_TSB_HUGE 1
-#define MM_TSB_XLHUGE 2
-
#ifndef __ASSEMBLY__
#define TSB_ENTRY_ALIGNMENT 16
unsigned long tsb_map_pte;
};
+#define MM_TSB_BASE 0
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-/* This is for the tsbs.*/
-#define MM_NUM_TSBS 3
-/* This is the count of huge_pte_count array. */
-#define MM_NUM_HUGEPAGE_SIZES 2
-#define MM_PTES_HUGE 0
-#define MM_PTES_XLHUGE 1
-
+#define MM_TSB_HUGE 1
+#define MM_NUM_TSBS 2
#else
#define MM_NUM_TSBS 1
-#define MM_NUM_HUGEPAGE_SIZES 0
#endif
typedef struct {
struct tsb_config tsb_block[MM_NUM_TSBS];
struct hv_tsb_descr tsb_descr[MM_NUM_TSBS];
void *vdso;
- unsigned long huge_pte_count[MM_NUM_HUGEPAGE_SIZES];
+ unsigned long hugetlb_pte_count;
unsigned long thp_pte_count;
bool adi;
} mm_context_t;
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-
-static inline unsigned long xl_hugepage_pte_count(mm_context_t *mm_context)
-{
- return mm_context->huge_pte_count[MM_PTES_XLHUGE];
-}
-
-static inline unsigned long hugepage_pte_count(mm_context_t *mm_context)
-{
- return mm_context->huge_pte_count[MM_PTES_HUGE];
-}
-
-static inline unsigned int hugepage_size_to_pte_count_idx(
- unsigned long hugepage_size)
-{
- unsigned int pte_count_index = MM_PTES_HUGE;
-
- if (hugepage_size != HPAGE_SIZE)
- pte_count_index = MM_PTES_XLHUGE;
-
- return pte_count_index;
-}
-
-void __init hv_establish_xl_hugepage_tsb_descriptor(unsigned short pgsz_idx,
- unsigned int pgsz_mask);
-
-#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */
-
-
#endif /* !__ASSEMBLY__ */
#define TSB_CONFIG_TSB 0x00
void __tsb_context_switch(unsigned long pgd_pa,
struct tsb_config *tsb_base,
struct tsb_config *tsb_huge,
- struct tsb_config *tsb_xl_huge,
unsigned long tsb_descr_pa);
static inline void tsb_context_switch(struct mm_struct *mm)
__tsb_context_switch(__pa(mm->pgd),
&mm->context.tsb_block[0],
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- (mm->context.tsb_block[MM_TSB_HUGE].tsb ?
- &mm->context.tsb_block[MM_TSB_HUGE] :
- NULL),
- (mm->context.tsb_block[MM_TSB_XLHUGE].tsb ?
- &mm->context.tsb_block[MM_TSB_XLHUGE] :
+ (mm->context.tsb_block[1].tsb ?
+ &mm->context.tsb_block[1] :
NULL)
#else
- NULL, NULL
+ NULL
#endif
, __pa(&mm->context.tsb_descr[0]));
}
#define HPAGE_SHIFT 23
#define REAL_HPAGE_SHIFT 22
-#define XLHPAGE_16GB_SHIFT 34
-#define XLHPAGE_2GB_SHIFT 31
-
+#define HPAGE_256MB_SHIFT 28
#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT)
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
#define HPAGE_MASK (~(HPAGE_SIZE - 1UL))
#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-#define HUGE_MAX_HSTATE 2
#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT))
+#define HUGE_MAX_HSTATE 2
#endif
#ifndef __ASSEMBLY__
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
struct pt_regs;
-void hugetlb_setup(struct pt_regs *regs, unsigned int tsb_index);
-extern unsigned int xl_hugepage_shift;
+void hugetlb_setup(struct pt_regs *regs);
#endif
#define WANT_PAGE_VIRTUAL
static inline pte_t pte_mkhuge(pte_t pte)
{
- return pte;
+ return __pte(pte_val(pte) | __pte_default_huge_mask());
}
static inline bool is_default_hugetlb_pte(pte_t pte)
#define pgd_page_vaddr(pgd) \
((unsigned long) __va(pgd_val(pgd)))
#define pgd_present(pgd) (pgd_val(pgd) != 0U)
-#define pgd_clear(pgdp) (pgd_val(*(pgd)) = 0UL)
+#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL)
static inline unsigned long pud_large(pud_t pud)
{
/* Actual page table PTE updates. */
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
- pte_t *ptep, pte_t orig, int fullmm);
+ pte_t *ptep, pte_t orig, int fullmm,
+ unsigned int hugepage_shift);
static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
- pte_t *ptep, pte_t orig, int fullmm)
+ pte_t *ptep, pte_t orig, int fullmm,
+ unsigned int hugepage_shift)
{
/* It is more efficient to let flush_tlb_kernel_range()
* handle init_mm tlb flushes.
* and SUN4V pte layout, so this inline test is fine.
*/
if (likely(mm != &init_mm) && pte_accessible(mm, orig))
- tlb_batch_add(mm, vaddr, ptep, orig, fullmm);
+ tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift);
}
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
pte_t orig = *ptep;
*ptep = pte;
- maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm);
+ maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT);
}
#define set_pte_at(mm,addr,ptep,pte) \
#define SCRATCHPAD_UTSBREG1 0x10
#define SCRATCHPAD_UTSBREG2 0x18
/* 0x20 and 0x28, hypervisor only... */
-#define SCRATCHPAD_XLHUGPAGES 0x30 /* Reserved for XL hugepages TSB */
+#define SCRATCHPAD_UNUSED1 0x30
#define SCRATCHPAD_UNUSED2 0x38 /* Reserved for OBP */
#endif /* !(_SPARC64_SCRATCHPAD_H) */
#define TLB_BATCH_NR 192
struct tlb_batch {
- bool default_huge;
+ unsigned int hugepage_shift;
struct mm_struct *mm;
unsigned long tlb_nr;
unsigned long active;
void flush_tsb_kernel_range(unsigned long start, unsigned long end);
void flush_tsb_user(struct tlb_batch *tb);
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge);
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr,
+ unsigned int hugepage_shift);
/* TLB flush operations. */
* page size in question. So for PMD mappings (which fall on
* bit 23, for 8MB per PMD) we must propagate bit 22 for a
* 4MB huge page. For huge PUDs (which fall on bit 33, for
- * 8GB per PUD), we have to accomodate 256MB and 2GB huge
+ * 8GB per PUD), we have to accommodate 256MB and 2GB huge
* pages. So for those we propagate bits 32 to 28.
*/
#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
* if it is a HUGE PMD or a normal one. If it is not valid
* then jump to FAIL_LABEL. If it is a HUGE PMD, and it
* translates to a valid PTE, branch to PTE_LABEL.
+ *
+ * We have to propagate the 4MB bit of the virtual address
+ * because we are fabricating 8MB pages using 4MB hw pages.
*/
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
sllx REG2, 32, REG2; \
andcc REG1, REG2, %g0; \
be,pt %xcc, 700f; \
- nop; \
+ sethi %hi(4 * 1024 * 1024), REG2; \
brgez,pn REG1, FAIL_LABEL; \
- nop; \
- ba PTE_LABEL; \
- nop; \
+ andn REG1, REG2, REG1; \
+ and VADDR, REG2, REG2; \
+ brlz,pt REG1, PTE_LABEL; \
+ or REG1, REG2, REG1; \
700:
#else
#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
* tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
* tsb_ptr = tsb_base + (tsb_index * 16);
*/
-#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, \
- PATCH_PROLOGUE, PATCH_EPILOGUE, \
- HASH_SHIFT, TMP1, TMP2) \
+#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
and TSB_PTR, 0x7, TMP1; \
mov 512, TMP2; \
andn TSB_PTR, 0x7, TSB_PTR; \
sllx TMP2, TMP1, TMP2; \
- PATCH_PROLOGUE; \
srlx VADDR, HASH_SHIFT, TMP1; \
- PATCH_EPILOGUE; \
sub TMP2, 1, TMP2; \
and TMP1, TMP2, TMP1; \
sllx TMP1, 4, TMP1; \
add TSB_PTR, TMP1, TSB_PTR;
- /* This is for xl_hugepages.*/
-#define PATCH_HASH_SHIFT_PROLOGUE \
- 661:;
-#define PATCH_HASH_SHIFT_EPILOGUE \
- .section .sun4v_xl_hugepage_hash_patch, "ax"; \
- .word 661b; \
- .previous;
-
- /* This is the normal tsb miss case.*/
-#define PATCH_HASH_SHIFT_NOPROLOGUE
-#define PATCH_HASH_SHIFT_NOEPILOGUE
-
sun4v_itlb_miss:
/* Load MMU Miss base into %g2. */
ldxa [%g0] ASI_SCRATCHPAD, %g2
LOAD_ITLB_INFO(%g2, %g4, %g5)
COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
- COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \
- PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g3, %g7)
+ COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
/* Load TSB tag/pte into %g2/%g3 and compare the tag. */
ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
LOAD_DTLB_INFO(%g2, %g4, %g5)
COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
- COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \
- PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g3, %g7)
+ COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
/* Load TSB tag/pte into %g2/%g3 and compare the tag. */
ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
/* fallthrough */
sun4v_tsb_miss_common:
- COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \
- PATCH_HASH_SHIFT_NOEPILOGUE, PAGE_SHIFT, %g5, %g7)
+ COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2
cmp %g5, -1
be,pt %xcc, 80f
nop
- COMPUTE_TSB_PTR(%g5, %g4, PATCH_HASH_SHIFT_NOPROLOGUE, \
- PATCH_HASH_SHIFT_NOEPILOGUE, REAL_HPAGE_SHIFT, %g2, %g7)
+ COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
/* That clobbered %g2, reload it. */
ldxa [%g0] ASI_SCRATCHPAD, %g2
ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath
ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
-#ifdef CONFIG_HUGETLB_PAGE
- /*
- * %g3 -- FAULT_CODE_{D,I}TLB
- * %g4 -- virtual address
- * %g5 -- pte
- * %g6 -- tag
- */
- .global sun4v_xl_hugepages
-sun4v_xl_hugepages:
- sethi %uhi(_PAGE_SPECIAL_4V), %g2
- sllx %g2, 32, %g2
- andcc %g5, %g2, %g0
- be,pt %xcc, 10f
- sethi %uhi(_PAGE_VALID | _PAGE_PMD_HUGE | _PAGE_SPECIAL_4V), %g1;
- sllx %g1, 32, %g1
- or %g1, _PAGE_SZALL_4V, %g1
- andn %g5, %g1, %g1
- ldxa [%g1 + %g0] ASI_PHYS_USE_EC, %g5
- brgez,pn %g5, tsb_do_fault
-10: mov SCRATCHPAD_XLHUGPAGES, %g1
- ldxa [%g1] ASI_SCRATCHPAD, %g1
- cmp %g1, -1
- beq,pn %xcc, 10f
- COMPUTE_TSB_PTR(%g1, %g4, PATCH_HASH_SHIFT_PROLOGUE, \
- PATCH_HASH_SHIFT_EPILOGUE, 0, %g2, %g7)
- ba,pt %xcc, tsb_reload
- nop
-10: SET_GL(1)
- rdpr %tl, %g7
- cmp %g7, 1
- bne,pn %xcc, winfix_trampoline
- mov %g3, %g4
- ba,pt %xcc, etrap
- rd %pc, %g7
- or %g0, MM_TSB_XLHUGE, %o1
- call hugetlb_setup
- add %sp, PTREGS_OFF, %o0
- ba,pt %xcc, rtrap
- nop
-#endif /* CONFIG_HUGETLB_PAGE */
-
sun4v_itlb_error:
rdpr %tl, %g1
cmp %g1, 1
*/
tsb_miss_dtlb:
mov TLB_TAG_ACCESS, %g4
+ ldxa [%g4] ASI_DMMU, %g4
+ srlx %g4, PAGE_SHIFT, %g4
ba,pt %xcc, tsb_miss_page_table_walk
- ldxa [%g4] ASI_DMMU, %g4
+ sllx %g4, PAGE_SHIFT, %g4
tsb_miss_itlb:
mov TLB_TAG_ACCESS, %g4
+ ldxa [%g4] ASI_IMMU, %g4
+ srlx %g4, PAGE_SHIFT, %g4
ba,pt %xcc, tsb_miss_page_table_walk
- ldxa [%g4] ASI_IMMU, %g4
+ sllx %g4, PAGE_SHIFT, %g4
/* At this point we have:
* %g1 -- PAGE_SIZE TSB entry address
/* Valid PTE is now in %g5. */
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-661: sethi %uhi(_PAGE_SZALL_4U), %g7
+ sethi %uhi(_PAGE_PMD_HUGE), %g7
sllx %g7, 32, %g7
- .section .sun4v_2insn_patch, "ax"
- .word 661b
- mov _PAGE_SZALL_4V, %g7
- nop
- .previous
- and %g5, %g7, %g2
-
- /* This deserves a comment. Should xl_hugepages be selected for
- * sun4v the patch order must be for correctness/safety reasons:
- * sun4v_xl_hugepage_hash_patch
- * sun4v_xl_hugepage_pte_size_patch
- * sun4v_xl_hugepage_pte_branch_patch
- * . Doing otherwise could result in hangs and thus boot failures.
- */
-661: sethi %uhi(_PAGE_SZHUGE_4U), %g7
-662: sllx %g7, 32, %g7
- .section .sun4v_2insn_patch, "ax"
- .word 661b
- mov _PAGE_SZHUGE_4V, %g7
- nop
- .previous
- .section .sun4v_xl_hugepage_pte_size_patch, "ax"
- .word 662b
- subcc %g2, 0, %g0
- .previous
-
- /* Should it be patched for xl_hugepages, then we need to fix up
- * the disp19 target to sun4v_xl_hugepages.
- */
-661:
+ andcc %g5, %g7, %g0
+ be,pt %xcc, 60f
nop
- .section .sun4v_xl_hugepage_pte_branch_patch, "ax"
- .word 661b
- beq,pn %xcc, 662f
-662:
- .previous
- cmp %g2, %g7
- bne,pt %xcc, 60f
- nop
-
- /* It is the default huge page (8M). We have to propagate
- * the 4MB bit of the virtual address because we are
- * fabricating 8MB pages using 4MB hw pages.
- */
- sethi %hi(4 * 1024 * 1024), %g2
- andn %g5, %g2, %g5
- and %g4, %g2, %g2
- or %g5, %g2, %g5
/* It is a huge page, use huge page TSB entry address we
* calculated above. If the huge page TSB has not been
mov %g3, %g4
ba,pt %xcc, etrap
rd %pc, %g7
- or %g0, MM_TSB_HUGE, %o1
call hugetlb_setup
add %sp, PTREGS_OFF, %o0
ba,pt %xcc, rtrap
nop
60:
-#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */
+#endif
/* At this point we have:
* %g1 -- TSB entry address
nop
.previous
+ /* Clear context ID bits. */
+ srlx %g5, PAGE_SHIFT, %g5
+ sllx %g5, PAGE_SHIFT, %g5
+
be,pt %xcc, sparc64_realfault_common
mov FAULT_CODE_DTLB, %g4
ba,pt %xcc, winfix_trampoline
* %o0: page table physical address
* %o1: TSB base config pointer
* %o2: TSB huge config pointer, or NULL if none
- * %o3 TSB XL huge config pointer or NULL if none
- * %o4: Hypervisor TSB descriptor physical address
+ * %o3: Hypervisor TSB descriptor physical address
*
* We have to run this whole thing with interrupts
* disabled so that the current cpu doesn't change
stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR]
ldx [%o1 + TSB_CONFIG_REG_VAL], %o0
- /* Check hugepage tsb */
brz,pt %o2, 1f
mov -1, %g3
mov SCRATCHPAD_UTSBREG2, %o5
stxa %g3, [%o5] ASI_SCRATCHPAD
- /* Start counting HV tsb descriptors. */
- mov 1, %o0
- cmp %g3, -1
- beq %xcc, 2f
- nop
- add %o0, 1, %o0
-2:
-
- /* check xl_hugepage tsb */
- brz,pt %o3, 3f
- mov -1, %g3
- ldx [%o3 + TSB_CONFIG_REG_VAL], %g3
-3:
- mov SCRATCHPAD_XLHUGPAGES, %o5
- stxa %g3, [%o5] ASI_SCRATCHPAD
-
+ mov 2, %o0
cmp %g3, -1
- beq %xcc, 4f
- nop
- add %o0, 1, %o0
-4:
+ move %xcc, 1, %o0
mov HV_FAST_MMU_TSB_CTXNON0, %o5
- mov %o4, %o1
+ mov %o3, %o1
ta HV_FAST_TRAP
/* Finish up. */
- ba,pt %xcc, 60f
+ ba,pt %xcc, 9f
nop
/* SUN4U TSB switch. */
stxa %o0, [%o5] ASI_IMMU
membar #Sync
- ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4
- brz %o4, 60f
+2: ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4
+ brz %o4, 9f
ldx [%o1 + TSB_CONFIG_MAP_PTE], %o5
sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2
stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS
membar #Sync
- brz,pt %o2, 60f
+ brz,pt %o2, 9f
nop
ldx [%o2 + TSB_CONFIG_MAP_VADDR], %o4
stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS
membar #Sync
-60:
+9:
wrpr %g1, %pstate
retl
*(.swapper_4m_tsb_phys_patch)
__swapper_4m_tsb_phys_patch_end = .;
}
- .sun4v_xl_hugepage_pte_size_patch : {
- __sun4v_xl_hugepage_pte_size_patch = .;
- *(.sun4v_xl_hugepage_pte_size_patch)
- __sun4v_xl_hugepage_pte_size_patch_end = .;
- }
- .sun4v_xl_hugepage_hash_patch : {
- __sun4v_xl_hugepage_hash_patch = .;
- *(.sun4v_xl_hugepage_hash_patch)
- __sun4v_xl_hugepage_hash_patch_end = .;
- }
- .sun4v_xl_hugepage_pte_branch_patch : {
- __sun4v_xl_hugepage_pte_branch_patch = .;
- *(.sun4v_xl_hugepage_pte_branch_patch)
- __sun4v_xl_hugepage_pte_branch_patch_end = .;
- }
.popc_3insn_patch : {
__popc_3insn_patch = .;
*(.popc_3insn_patch)
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
if (is_hugetlb_pmd(*pmdp)) {
- unsigned long hpage_mask = HPAGE_MASK;
-
- if (xl_hugepage_shift == XLHPAGE_2GB_SHIFT)
- hpage_mask = ~((1UL << xl_hugepage_shift) - 1);
- if (pmd_trans_splitting(*pmdp))
- goto out_irq_enable;
-
pa = pmd_pfn(*pmdp) << PAGE_SHIFT;
- pa += tpc & ~hpage_mask;
+ pa += tpc & ~HPAGE_MASK;
/* Use phys bypass so we don't pollute dtlb/dcache. */
__asm__ __volatile__("lduwa [%1] %2, %0"
show_regs(regs);
}
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-/* Put this here until there are more consumers.*/
-static void sparc64_hugetlb_tsb_fault(struct pt_regs *regs,
- struct mm_struct *mm,
- unsigned int hugepage_shift)
-{
- unsigned int hugepage_pte_idx, hugepage_idx;
- unsigned long mm_rss;
-
- if (hugepage_shift == xl_hugepage_shift)
- hugepage_idx = MM_TSB_XLHUGE;
- else
- hugepage_idx = MM_TSB_HUGE;
-
- hugepage_pte_idx =
- hugepage_size_to_pte_count_idx(1UL << hugepage_shift);
-
- mm_rss = mm->context.huge_pte_count[hugepage_pte_idx];
- if (hugepage_idx == MM_TSB_HUGE) {
-#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
- mm_rss += mm->context.thp_pte_count;
-#endif
- mm_rss *= REAL_HPAGE_PER_HPAGE;
- }
-
- if (unlikely(mm_rss >
- mm->context.tsb_block[hugepage_idx].tsb_rss_limit)) {
- if (mm->context.tsb_block[hugepage_idx].tsb)
- tsb_grow(mm, hugepage_idx, mm_rss);
- else
- hugetlb_setup(regs, hugepage_idx);
-
- }
-}
-#else
-static void sparc64_hugetlb_tsb_fault(struct pt_regs *regs,
- struct mm_struct *mm,
- unsigned int hugepage_shift)
-{
-}
-#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */
-
asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
{
enum ctx_state prev_state = exception_enter();
struct mm_struct *mm = current->mm;
- unsigned int hugepage_shift;
struct vm_area_struct *vma;
unsigned int insn = 0;
int si_code, fault_code, fault;
goto retry;
}
}
- if (is_vm_hugetlb_page(vma))
- hugepage_shift = huge_page_shift(hstate_vma(vma));
- else
- hugepage_shift = HPAGE_SHIFT;
up_read(&mm->mmap_sem);
mm_rss = get_mm_rss(mm);
if (unlikely(mm_rss >
mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
tsb_grow(mm, MM_TSB_BASE, mm_rss);
- sparc64_hugetlb_tsb_fault(regs, mm, hugepage_shift);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
+ mm_rss *= REAL_HPAGE_PER_HPAGE;
+ if (unlikely(mm_rss >
+ mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
+ if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
+ tsb_grow(mm, MM_TSB_HUGE, mm_rss);
+ else
+ hugetlb_setup(regs);
+
+ }
+#endif
exit_exception:
exception_exit(prev_state);
return;
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/sysctl.h>
+
#include <asm/mman.h>
#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
-#include <asm/pgtable.h>
/* Slightly simplified from the non-hugepage variant because by
* definition we don't have to worry about any page coloring stuff
*/
-static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
+
+static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
unsigned long addr,
unsigned long len,
unsigned long pgoff,
unsigned long flags)
{
- struct hstate *h = hstate_file(file);
+ struct hstate *h = hstate_file(filp);
unsigned long task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
}
static unsigned long
-hugetlb_get_unmapped_area_topdown(struct file *file,
- const unsigned long addr0,
+hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len,
const unsigned long pgoff,
const unsigned long flags)
{
- struct hstate *h = hstate_file(file);
+ struct hstate *h = hstate_file(filp);
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
struct vm_unmapped_area_info info;
return addr;
}
-unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
- unsigned long len, unsigned long pgoff,
- unsigned long flags)
+unsigned long
+hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct hstate *h = hstate_file(file);
- unsigned long task_size = TASK_SIZE;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
+ unsigned long task_size = TASK_SIZE;
if (test_thread_flag(TIF_32BIT))
task_size = STACK_TOP32;
pgoff, flags);
}
-/* Since the hugepage could cover more than one pmd entry and more
- * than one pgd entry we must cover all possible conditions.
- */
-static pmd_t *huge_pmd_alloc(struct mm_struct *mm, unsigned long addr)
-{
- pgd_t *pgd = pgd_offset(mm, addr);
- pmd_t *pmd = NULL;
- pud_t *pud;
-
- if (pgd_none(*pgd)) {
- pud_t *pud = pud_alloc(mm, pgd, addr);
-
- if (pud == NULL)
- goto out;
- }
- pud = pud_offset(pgd, addr);
- if (pud_none(*pud)) {
- pmd = pmd_alloc(mm, pud, addr);
-
- if (pmd == NULL)
- goto out;
- }
- pmd = pmd_offset(pud, addr);
-out:
- return pmd;
-}
-
-/* Note, should we fail leave behind the mm state
- * which will be cleaned up on exit.
- */
-pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr,
- unsigned long size)
-{
- unsigned long start = addr & ~(size - 1);
- unsigned long end = start + size;
- pte_t *rpte = NULL;
-
- /* Our caller operates on start's pte which is rpte should we succeed.*/
- for (addr = start; addr < end; addr = addr + PMD_SIZE) {
- pmd_t *pmd = huge_pmd_alloc(mm, addr);
- pte_t *pte;
-
- if (!pmd)
- goto fail;
- if (size != (1UL << XLHPAGE_16GB_SHIFT)) {
- rpte = (pte_t *)pmd;
- break;
- }
-
- pte = pte_alloc_map(mm, NULL, pmd, addr);
-
- if (!pte)
- goto fail;
- else if (!rpte)
- rpte = pte;
- }
-
- return rpte;
-fail:
- return NULL;
-}
-
int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
{
return 0;
}
-/* This function possibly needs to be moved. It will be different
- * for sun4u and even possibly for sun4v future cores. Though we have
- * no plans to support sun4u at this point.
- */
-static unsigned int sun4v_tte_to_shift(pte_t entry)
-{
- unsigned long hugepage_tte = pte_val(entry) & _PAGE_SZALL_4V;
- unsigned int hugepage_shift;
-
- switch (hugepage_tte) {
- case _PAGE_SZ16GB_4V:
- hugepage_shift = XLHPAGE_16GB_SHIFT;
- break;
- case _PAGE_SZ2GB_4V:
- hugepage_shift = XLHPAGE_2GB_SHIFT;
- break;
- case _PAGE_SZ4MB_4V:
- hugepage_shift = REAL_HPAGE_SHIFT;
- break;
- default:
- WARN_ONCE(1, "hugepage_shift: hugepage_tte=0x%lx\n",
- hugepage_tte);
- hugepage_shift = PAGE_SHIFT;
- break;
- }
- return hugepage_shift;
-}
-
-static unsigned int tte_to_shift(pte_t entry)
-{
- unsigned int hugepage_shift;
-
- if (tlb_type == hypervisor)
- hugepage_shift = sun4v_tte_to_shift(entry);
- else
- hugepage_shift = REAL_HPAGE_SHIFT;
-
- return hugepage_shift;
-}
-
-static unsigned long tte_to_hugepage_size(pte_t pte)
+static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
{
- unsigned long hugepage_size = 1UL << tte_to_shift(pte);
-
- if (hugepage_size == REAL_HPAGE_SIZE)
- hugepage_size = HPAGE_SIZE;
- return hugepage_size;
-}
-
-static unsigned long tte_to_hugepage_mask(pte_t pte)
-{
- unsigned int hugepage_shift = tte_to_shift(pte);
- unsigned long hugepage_mask;
-
- if (hugepage_shift == REAL_HPAGE_SHIFT)
- hugepage_shift = HPAGE_SHIFT;
-
- hugepage_mask = ~((1UL << hugepage_shift) - 1);
-
- return hugepage_mask;
+ return entry;
}
-/* This should also be moved and a noop for sun4u.
- * Only include hugepage sizes we plan to support.
- */
-static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int hugepage_shift)
+static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
{
- unsigned long sun4v_hugepage_size = _PAGE_SZ4MB_4V;
+ unsigned long hugepage_size = _PAGE_SZ4MB_4V;
pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
- switch (hugepage_shift) {
- /* 16Gb */
- case XLHPAGE_16GB_SHIFT:
- sun4v_hugepage_size = _PAGE_SZ16GB_4V;
- break;
- /* 2Gb */
- case XLHPAGE_2GB_SHIFT:
- sun4v_hugepage_size = _PAGE_SZ2GB_4V;
+ switch (shift) {
+ case HPAGE_256MB_SHIFT:
+ hugepage_size = _PAGE_SZ256MB_4V;
pte_val(entry) |= _PAGE_PMD_HUGE;
break;
- /* 8Mb */
case HPAGE_SHIFT:
pte_val(entry) |= _PAGE_PMD_HUGE;
break;
default:
- WARN_ONCE(hugepage_shift,
- "hugepage_shift_to_tte: unsupported "
- "hugepage_shift=%u.\n", hugepage_shift);
+ WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
}
- pte_val(entry) = pte_val(entry) | sun4v_hugepage_size;
+ pte_val(entry) = pte_val(entry) | hugepage_size;
return entry;
}
+static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
+{
+ if (tlb_type == hypervisor)
+ return sun4v_hugepage_shift_to_tte(entry, shift);
+ else
+ return sun4u_hugepage_shift_to_tte(entry, shift);
+}
+
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
struct page *page, int writeable)
{
return hugepage_shift_to_tte(entry, hugepage_shift);
}
-static void huge_pte_at_flush_update(struct mm_struct *mm, unsigned long addr,
- pte_t *pte, pte_t orig,
- pte_t *sentinel_pte)
-{
- if (pte_val(orig) & _PAGE_VALID) {
- if (!(pte_val(*sentinel_pte) & _PAGE_VALID)) {
- *sentinel_pte = orig;
- tlb_batch_add(mm, addr, pte, orig, false);
- }
- }
-}
-
-static void form_sentinel(pte_t *sentinel_pte, pte_t entry, pte_t *pte,
- unsigned int hugepage_shift)
-{
- pte_t sentinel = __pte(_PAGE_VALID | _PAGE_SPECIAL_4V |
- (pte_val(entry) & _PAGE_SZALL_4V) | __pa(pte));
-
- BUG_ON(__pa(pte) & _PAGE_SZALL_4V);
- if (hugepage_shift == 31U) /* 2G page */
- sentinel = __pte(pte_val(sentinel) | _PAGE_PMD_HUGE);
-
- *sentinel_pte = sentinel;
-}
-
-static bool huge_pte_at_handle_sentinel(pte_t *sentinel_pte, pte_t *pte,
- pte_t orig, pte_t entry,
- unsigned int hugepage_shift)
+static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
{
- bool rc = true;
+ unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
+ unsigned int shift;
- /* Should the original pte be marked valid then
- * only update the sentinel.
- */
- if (pte_val(orig) & _PAGE_VALID) {
- if ((pte_val(orig) & _PAGE_SPECIAL_4V) == 0UL)
- *pte = entry;
- rc = false;
- } else if (pte_val(*sentinel_pte) & _PAGE_VALID) {
- *pte = *sentinel_pte;
- } else {
- form_sentinel(sentinel_pte, entry, pte, hugepage_shift);
- *pte = entry;
+ switch (tte_szbits) {
+ case _PAGE_SZ256MB_4V:
+ shift = HPAGE_256MB_SHIFT;
+ break;
+ case _PAGE_SZ4MB_4V:
+ shift = REAL_HPAGE_SHIFT;
+ break;
+ default:
+ shift = PAGE_SHIFT;
+ break;
}
-
- return rc;
+ return shift;
}
-static bool __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *pte, pte_t entry, pte_t *sentinel_pte,
- unsigned int hugepage_shift)
+static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
{
- bool rc = true;
-
- if (hugepage_shift != REAL_HPAGE_SHIFT) {
- pte_t orig = *pte;
+ unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
+ unsigned int shift;
- rc = huge_pte_at_handle_sentinel(sentinel_pte, pte, orig,
- entry, hugepage_shift);
- huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte);
- } else
- *pte = entry;
-
- return rc;
+ switch (tte_szbits) {
+ case _PAGE_SZ256MB_4U:
+ shift = HPAGE_256MB_SHIFT;
+ break;
+ case _PAGE_SZ4MB_4U:
+ shift = REAL_HPAGE_SHIFT;
+ break;
+ default:
+ shift = PAGE_SHIFT;
+ break;
+ }
+ return shift;
}
-static void __clear_huge_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *pte, pte_t *sentinel_pte,
- unsigned int hugepage_shift)
+static unsigned int huge_tte_to_shift(pte_t entry)
{
- if (hugepage_shift != REAL_HPAGE_SHIFT) {
- pte_t orig = *pte;
+ unsigned long shift;
- *pte = __pte(0UL);
- huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte);
- } else
- *pte = __pte(0UL);
-}
-
-static bool set_huge_pte_range_at(struct mm_struct *mm, pmd_t *pmd,
- unsigned long addr, pte_t *pentry,
- pte_t *sentinel_pte, bool set_at)
-{
- pte_t *pte = pte_offset_map(pmd, addr);
- pte_t *lpte = pte + PTRS_PER_PTE;
- pte_t entry = *pentry;
- pte_t orig = *(pte_t *)pte;
- bool rc = true;
- unsigned long orig_addr = addr;
- unsigned int hugepage_shift;
-
- if (set_at)
- hugepage_shift = tte_to_shift(entry);
+ if (tlb_type == hypervisor)
+ shift = sun4v_huge_tte_to_shift(entry);
else
- hugepage_shift = tte_to_shift(*pte);
-
- for (; pte < lpte; pte++, addr = addr + PAGE_SIZE) {
- if (set_at) {
- rc = __set_huge_pte_at(mm, addr, pte, entry,
- sentinel_pte, hugepage_shift);
- if (!rc)
- break;
- pte_val(entry) = pte_val(entry) + PAGE_SIZE;
- } else
- __clear_huge_pte_at(mm, addr, pte, sentinel_pte,
- hugepage_shift);
- }
- if (set_at)
- *pentry = entry;
-
- if (hugepage_shift == REAL_HPAGE_SHIFT) {
- /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
- maybe_tlb_batch_add(mm, orig_addr, pte, orig, 0);
- maybe_tlb_batch_add(mm, orig_addr + REAL_HPAGE_SIZE,
- pte, orig, 0);
- }
+ shift = sun4u_huge_tte_to_shift(entry);
- return rc;
+ if (shift == PAGE_SHIFT)
+ WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
+ pte_val(entry));
+
+ return shift;
}
-static bool __set_huge_pmd_at(struct mm_struct *mm, pmd_t *pmd,
- unsigned long addr, unsigned long end, pte_t *pentry,
- pte_t *sentinel_pte, bool set_at)
+static unsigned long huge_tte_to_size(pte_t pte)
{
- bool rc;
- pte_t orig;
- pte_t entry;
- unsigned long next;
- unsigned long hugepage_shift;
-
- rc = true;
- orig = *(pte_t *)pmd;
- entry = *pentry;
-
- if (set_at) {
- hugepage_shift = tte_to_shift(entry);
- if (hugepage_shift == REAL_HPAGE_SHIFT) {
- *pmd = __pmd(pte_val(entry));
- } else {
- do {
- next = pmd_addr_end(addr, end);
- rc = __set_huge_pte_at(mm, addr, (pte_t *)pmd,
- entry, sentinel_pte, hugepage_shift);
- if (!rc)
- break;
- } while (pmd++, addr = next, addr != end);
- }
- *pentry = entry;
- } else {
- hugepage_shift = tte_to_shift(orig);
- if (hugepage_shift == REAL_HPAGE_SHIFT) {
- *pmd = __pmd(0);
- } else {
- do {
- next = pmd_addr_end(addr, end);
- __clear_huge_pte_at(mm, addr, (pte_t *)pmd,
- sentinel_pte, hugepage_shift);
- } while (pmd++, addr = next, addr != end);
- }
- }
-
- if (hugepage_shift == REAL_HPAGE_SHIFT) {
- /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
- maybe_tlb_batch_add(mm, addr, (pte_t *)pmd, orig, 0);
- maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE,
- (pte_t *)pmd, orig, 0);
- }
+ unsigned long size = 1UL << huge_tte_to_shift(pte);
- return rc;
+ if (size == REAL_HPAGE_SIZE)
+ size = HPAGE_SIZE;
+ return size;
}
-static bool set_huge_pmd_at(struct mm_struct *mm, pud_t *pud,
- unsigned long addr, unsigned long end,
- pte_t *pentry, pte_t *sentinel_pte, bool set_at)
+pte_t *huge_pte_alloc(struct mm_struct *mm,
+ unsigned long addr, unsigned long sz)
{
- pmd_t *pmd = pmd_offset(pud, addr);
- unsigned long next;
- bool rc;
- unsigned int is_huge_pmd;
-
- if (set_at)
- is_huge_pmd = is_hugetlb_pmd(__pmd(pte_val(*pentry)));
- else
- is_huge_pmd = is_hugetlb_pmd(*pmd);
+ pgd_t *pgd;
+ pud_t *pud;
+ pte_t *pte = NULL;
- if (is_huge_pmd)
- return __set_huge_pmd_at(mm, pmd, addr, end, pentry,
- sentinel_pte, set_at);
+ pgd = pgd_offset(mm, addr);
+ pud = pud_alloc(mm, pgd, addr);
+ if (pud)
+ pte = (pte_t *)pmd_alloc(mm, pud, addr);
- do {
- next = pmd_addr_end(addr, end);
- rc = set_huge_pte_range_at(mm, pmd, addr, pentry,
- sentinel_pte, set_at);
- } while (pmd++, addr = next, ((addr != end) && rc));
- return rc;
+ return pte;
}
-static bool set_huge_pud_at(struct mm_struct *mm, pgd_t *pgd,
- unsigned long addr, unsigned long end,
- pte_t *pentry, pte_t *sentinel_pte, bool set_at)
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
- pud_t *pud = pud_offset(pgd, addr);
- unsigned long next;
- bool rc;
+ pgd_t *pgd;
+ pud_t *pud;
+ pte_t *pte = NULL;
- do {
- next = pud_addr_end(addr, end);
- rc = set_huge_pmd_at(mm, pud, addr, next, pentry,
- sentinel_pte, set_at);
- } while (pud++, addr = next, ((addr != end) && rc));
- return rc;
+ pgd = pgd_offset(mm, addr);
+ if (!pgd_none(*pgd)) {
+ pud = pud_offset(pgd, addr);
+ if (!pud_none(*pud))
+ pte = (pte_t *)pmd_offset(pud, addr);
+ }
+ return pte;
}
-/* entry must be the first pte of the hugepage. Otherwise entry
- * must be adjusted before we enter the loop for set_pte_at and
- * aligned physically to match the hugepage_size. This is equally
- * true of other locations where HUGETLB_PAGE_ORDER is used within
- * this module for mainline as of 7/20/2014.
- */
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
- pte_t sentinel_pte = __pte(0UL);
- unsigned long hugepage_size = tte_to_hugepage_size(entry);
- unsigned long hugepage_mask = tte_to_hugepage_mask(entry);
- unsigned long start = addr & hugepage_mask;
- unsigned long end = start + hugepage_size;
- pgd_t *pgd = pgd_offset(mm, start);
- unsigned long next;
- bool rc;
+ unsigned int i, nptes, hugepage_shift;
+ unsigned long size;
+ pte_t orig;
- if (!pte_present(*ptep) && pte_present(entry)) {
- unsigned int pte_count_idx =
- hugepage_size_to_pte_count_idx(hugepage_size);
+ size = huge_tte_to_size(entry);
+ nptes = size >> PMD_SHIFT;
- mm->context.huge_pte_count[pte_count_idx]++;
- }
+ if (!pte_present(*ptep) && pte_present(entry))
+ mm->context.hugetlb_pte_count += nptes;
- do {
- next = pgd_addr_end(start, end);
- rc = set_huge_pud_at(mm, pgd, start, next, &entry,
- &sentinel_pte, true);
- } while (pgd++, start = next, ((start != end) && rc));
+ addr &= ~(size - 1);
+ orig = *ptep;
+ hugepage_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
+
+ for (i = 0; i < nptes; i++)
+ ptep[i] = __pte(pte_val(entry) + (i << PMD_SHIFT));
+
+ maybe_tlb_batch_add(mm, addr, ptep, orig, 0, hugepage_shift);
+ /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
+ if (size == HPAGE_SIZE)
+ maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
+ hugepage_shift);
}
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
- pte_t sentinel_pte = __pte(0UL);
- pte_t entry = *ptep;
- unsigned long hugepage_size = tte_to_hugepage_size(entry);
- unsigned long hugepage_mask = tte_to_hugepage_mask(entry);
- unsigned long start = addr & hugepage_mask;
- unsigned long end = start + hugepage_size;
- pgd_t *pgd = pgd_offset(mm, start);
- unsigned long next;
- bool rc;
-
- if (pte_present(entry)) {
- unsigned int pte_count_idx =
- hugepage_size_to_pte_count_idx(hugepage_size);
-
- mm->context.huge_pte_count[pte_count_idx]--;
- }
+ unsigned int i, nptes, hugepage_shift;
+ unsigned long size;
+ pte_t entry;
- do {
- next = pgd_addr_end(start, end);
- rc = set_huge_pud_at(mm, pgd, start, next, &entry,
- &sentinel_pte, false);
- } while (pgd++, start = next, ((start != end) && rc));
+ entry = *ptep;
+ size = huge_tte_to_size(entry);
+ nptes = size >> PMD_SHIFT;
+ hugepage_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
- return entry;
-}
+ if (pte_present(entry))
+ mm->context.hugetlb_pte_count -= nptes;
-pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
-{
- pte_t *pte = NULL;
- pgd_t *pgd;
- pud_t *pud;
- pmd_t *pmd;
+ addr &= ~(size - 1);
+ for (i = 0; i < nptes; i++)
+ ptep[i] = __pte(0UL);
- pgd = pgd_offset(mm, addr);
- if (!pgd_none(*pgd)) {
- pud = pud_offset(pgd, addr);
- if (!pud_none(*pud)) {
- pmd = pmd_offset(pud, addr);
- if (xl_hugepage_shift != XLHPAGE_16GB_SHIFT)
- pte = (pte_t *)pmd;
- else if (!pmd_none(*pmd))
- pte = pte_offset_map(pmd, addr);
- }
- }
+ maybe_tlb_batch_add(mm, addr, ptep, entry, 0, hugepage_shift);
+ /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
+ if (size == HPAGE_SIZE)
+ maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
+ hugepage_shift);
- return pte;
+ return entry;
}
int pmd_huge(pmd_t pmd)
{
- return 0;
+ return !pmd_none(pmd) &&
+ (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID;
}
int pud_huge(pud_t pud)
}
#ifdef CONFIG_HUGETLB_PAGE
-unsigned int xl_hugepage_shift;
-static unsigned long xl_hugepage_pte;
-
-static bool is_xl_hugetlb_pte(pte_t pte)
-{
- bool rc = false;
-
- if (!xl_hugepage_pte)
- goto out;
- else if ((pte_val(pte) & _PAGE_SZALL_4V) == xl_hugepage_pte)
- rc = true;
-out:
- return rc;
-}
-
-static void __init sun4v_xl_hugepage_hash_patch(void)
-{
- extern unsigned int __sun4v_xl_hugepage_hash_patch;
- unsigned *insn, *p;
-
- p = &__sun4v_xl_hugepage_hash_patch;
- insn = (unsigned int *)(unsigned long)*p;
- *insn = *insn | xl_hugepage_shift;
- __asm__ __volatile__("flush %0\n\t"
- : /* no outputs */
- : "r" (insn));
-}
-
-static void __init sun4v_xl_hugepage_pte_size_patch(void)
-{
- extern unsigned int __sun4v_xl_hugepage_pte_size_patch;
- unsigned *insn, *p;
-
- p = &__sun4v_xl_hugepage_pte_size_patch;
- insn = (unsigned int *)(unsigned long)*p;
- p++;
- /* It is a simm13 in subcc instruction.*/
- *insn = *p | (unsigned int) xl_hugepage_pte;
- __asm__ __volatile__("flush %0\n\t"
- : /* no outputs */
- : "r" (insn));
-}
-
-static void __init sun4v_xl_hugepage_pte_branch_patch(void)
-{
- extern unsigned int __sun4v_xl_hugepage_pte_branch_patch;
- extern unsigned int sun4v_xl_hugepages;
- unsigned int btarget = (unsigned int)
- (unsigned long) &sun4v_xl_hugepages;
- unsigned int *insn, *p, disp19;
-
- p = &__sun4v_xl_hugepage_pte_branch_patch;
- insn = (unsigned int *)(unsigned long)*p;
- /* Instruction which needs to be a bne,pt to sun4v_xl_hugepages.*/
- p++;
- disp19 = (btarget - (unsigned int) (unsigned long) insn);
- disp19 = disp19 >> 2;
- disp19 = disp19 & (0x7ffff);
- *insn = *p & ~0x7ffff;
- *insn = *insn | disp19;
- __asm__ __volatile__("flush %0\n\t"
- : /* no outputs */
- : "r" (insn));
-}
-
-static int __init setup_xl_hugepage(unsigned int hugepage_shift)
+static int __init setup_hugepagesz(char *string)
{
+ unsigned long long hugepage_size;
+ unsigned int hugepage_shift;
unsigned short hv_pgsz_idx;
unsigned int hv_pgsz_mask;
int rc = 0;
+ hugepage_size = memparse(string, &string);
+ hugepage_shift = ilog2(hugepage_size);
+
switch (hugepage_shift) {
- case XLHPAGE_16GB_SHIFT:
- xl_hugepage_pte = _PAGE_SZ16GB_4V;
- hv_pgsz_mask = HV_PGSZ_MASK_16GB;
- hv_pgsz_idx = HV_PGSZ_IDX_16GB;
+ case HPAGE_256MB_SHIFT:
+ hv_pgsz_mask = HV_PGSZ_MASK_256MB;
+ hv_pgsz_idx = HV_PGSZ_IDX_256MB;
break;
- case XLHPAGE_2GB_SHIFT:
- xl_hugepage_pte = _PAGE_SZ2GB_4V;
- hv_pgsz_mask = HV_PGSZ_MASK_2GB;
- hv_pgsz_idx = HV_PGSZ_IDX_2GB;
+ case HPAGE_SHIFT:
+ hv_pgsz_mask = HV_PGSZ_MASK_4MB;
+ hv_pgsz_idx = HV_PGSZ_IDX_4MB;
break;
default:
- goto out;
+ hv_pgsz_mask = 0;
}
if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) {
- pr_warn("hugepagesz=%luM not supported by MMU.\n",
- (1UL << (hugepage_shift - 20)));
+ pr_warn("hugepagesz=%llu not supported by MMU.\n",
+ hugepage_size);
goto out;
}
- if (xl_hugepage_shift) {
- pr_warn("hugepagesz=%luM in use and hugepagesz=%luM"
- " ignored.\n",
- (1UL << (xl_hugepage_shift - 20)),
- (1UL << (hugepage_shift - 20)));
- goto out;
- }
-
- xl_hugepage_shift = hugepage_shift;
- sun4v_xl_hugepage_hash_patch();
- sun4v_xl_hugepage_pte_size_patch();
- sun4v_xl_hugepage_pte_branch_patch();
- hv_establish_xl_hugepage_tsb_descriptor(hv_pgsz_idx, hv_pgsz_mask);
- hugetlb_add_hstate(xl_hugepage_shift - PAGE_SHIFT);
+ hugetlb_add_hstate(hugepage_shift - PAGE_SHIFT);
rc = 1;
-out:
- return rc;
-}
-
-static int __init setup_hugepagesz(char *string)
-{
- unsigned long long xl_hugepage_size;
- int rc = 0;
-
- if (tlb_type != hypervisor)
- goto out;
-
- xl_hugepage_size = memparse(string, &string);
-
- switch (xl_hugepage_size) {
- case 1UL << XLHPAGE_16GB_SHIFT:
- case 1UL << XLHPAGE_2GB_SHIFT:
- {
- unsigned int hugepage_shift = ilog2(xl_hugepage_size);
-
- rc = setup_xl_hugepage(hugepage_shift);
- break;
- }
- default:
- pr_warn("hugepage size(0x%llx) not supported.\n",
- xl_hugepage_size);
- break;
- case HPAGE_SIZE:
- hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
- rc = 1;
- break;
- }
out:
return rc;
spin_lock_irqsave(&mm->context.lock, flags);
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if ((mm->context.huge_pte_count[MM_PTES_HUGE] ||
- mm->context.thp_pte_count) && is_default_hugetlb_pte(pte)) {
- /* We are fabricating 8MB pages using 4MB real hw pages */
+ if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) &&
+ is_hugetlb_pmd(__pmd(pte_val(pte)))) {
+ /* We are fabricating 8MB pages using 4MB real hw pages. */
pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT));
__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
address, pte_val(pte));
- } else if (mm->context.huge_pte_count[MM_PTES_XLHUGE] &&
- is_xl_hugetlb_pte(pte))
- __update_mmu_tsb_insert(mm, MM_TSB_XLHUGE, xl_hugepage_shift,
- address, pte_val(pte));
- else
+ } else
#endif
__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
address, pte_val(pte));
load_secondary_context(mm);
}
-void hugetlb_setup(struct pt_regs *regs, unsigned int tsb_index)
+void hugetlb_setup(struct pt_regs *regs)
{
struct mm_struct *mm = current->mm;
struct tsb_config *tp;
die_if_kernel("HugeTSB in atomic", regs);
}
- tp = &mm->context.tsb_block[tsb_index];
+ tp = &mm->context.tsb_block[MM_TSB_HUGE];
if (likely(tp->tsb == NULL))
- tsb_grow(mm, tsb_index, 0);
+ tsb_grow(mm, MM_TSB_HUGE, 0);
tsb_context_switch(mm);
smp_tsb_sync(mm);
}
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
- bool exec, bool default_huge)
+ bool exec, unsigned int hugepage_shift)
{
struct tlb_batch *tb = &get_cpu_var(tlb_batch);
unsigned long nr;
}
if (!tb->active) {
- flush_tsb_user_page(mm, vaddr, default_huge);
+ flush_tsb_user_page(mm, vaddr, hugepage_shift);
global_flush_tlb_page(mm, vaddr);
goto out;
}
if (nr == 0) {
tb->mm = mm;
- tb->default_huge = default_huge;
+ tb->hugepage_shift = hugepage_shift;
}
- if (tb->default_huge != default_huge) {
+ if (tb->hugepage_shift != hugepage_shift) {
flush_tlb_pending();
- tb->default_huge = default_huge;
+ tb->hugepage_shift = hugepage_shift;
nr = 0;
}
}
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
- pte_t *ptep, pte_t orig, int fullmm)
+ pte_t *ptep, pte_t orig, int fullmm,
+ unsigned int hugepage_shift)
{
- bool default_huge = is_default_hugetlb_pte(orig);
-
if (tlb_type != hypervisor &&
pte_dirty(orig)) {
unsigned long paddr, pfn = pte_pfn(orig);
no_cache_flush:
if (!fullmm)
- tlb_batch_add_one(mm, vaddr, pte_exec(orig), default_huge);
+ tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
if (pte_val(*pte) & _PAGE_VALID) {
bool exec = pte_exec(*pte);
- tlb_batch_add_one(mm, vaddr, exec, false);
+ tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);
}
pte++;
vaddr += PAGE_SIZE;
* for huge zero page. Huge zero pages are like hugetlb
* pages in that there is no RSS, but there is the need
* for TSB entries. So, huge zero page counts go into
- * huge_pte_count[MM_PTES_HUGE].
+ * hugetlb_pte_count.
*/
if (pmd_val(pmd) & _PAGE_PMD_HUGE) {
if (is_huge_zero_page(pmd_page(pmd)))
- mm->context.huge_pte_count[MM_PTES_HUGE]++;
+ mm->context.hugetlb_pte_count++;
else
mm->context.thp_pte_count++;
} else {
if (is_huge_zero_page(pmd_page(orig)))
- mm->context.huge_pte_count[MM_PTES_HUGE]--;
+ mm->context.hugetlb_pte_count--;
else
mm->context.thp_pte_count--;
}
pte_t orig_pte = __pte(pmd_val(orig));
bool exec = pte_exec(orig_pte);
- tlb_batch_add_one(mm, addr, exec, true);
+ tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
- true);
+ REAL_HPAGE_SHIFT);
} else {
tlb_batch_pmd_scan(mm, addr, orig);
}
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
/*
- * set_pmd_at() will not be called in a way to decrement the
- * context.thp_pte_count when splitting a THP, so do it now.
+ * set_pmd_at() will not be called in a way to decrement
+ * thp_pte_count when splitting a THP, so do it now.
* Sanity check pmd before doing the actual decrement.
*/
if ((pmd_val(entry) & _PAGE_PMD_HUGE) &&
!is_huge_zero_page(pmd_page(entry)))
(vma->vm_mm)->context.thp_pte_count--;
-
}
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
__flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries);
}
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v,
+ unsigned long hash_shift,
+ unsigned long nentries,
+ unsigned int hugepage_shift)
+{
+ unsigned int hpage_entries;
+ unsigned int i;
+
+ hpage_entries = 1 << (hugepage_shift - hash_shift);
+ for (i = 0; i < hpage_entries; i++)
+ __flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift,
+ nentries);
+}
+
+static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
+ unsigned long tsb, unsigned long nentries,
+ unsigned int hugepage_shift)
+{
+ unsigned long i;
+
+ for (i = 0; i < tb->tlb_nr; i++)
+ __flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift,
+ nentries, hugepage_shift);
+}
+#endif
+
void flush_tsb_user(struct tlb_batch *tb)
{
struct mm_struct *mm = tb->mm;
spin_lock_irqsave(&mm->context.lock, flags);
- if (!tb->default_huge) {
+ if (tb->hugepage_shift == PAGE_SHIFT) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
__flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
}
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (tb->default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
base = __pa(base);
- __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries);
+ __flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries,
+ tb->hugepage_shift);
}
#endif
-#ifdef CONFIG_HUGETLB_PAGE
- if (!tb->default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
- base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries;
- base = __pa(base);
- __flush_tsb_one(tb, xl_hugepage_shift, base, nentries);
- }
-#endif /* CONFIG_HUGETLB_PAGE */
spin_unlock_irqrestore(&mm->context.lock, flags);
}
void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr,
- bool default_huge)
+ unsigned int hugepage_shift)
{
unsigned long nentries, base, flags;
spin_lock_irqsave(&mm->context.lock, flags);
- if (!default_huge) {
+ if (hugepage_shift == PAGE_SHIFT) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
__flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
}
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
base = __pa(base);
- __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries);
+ __flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT,
+ nentries, hugepage_shift);
}
#endif
-#ifdef CONFIG_HUGETLB_PAGE
- if (!default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
- base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries;
- base = __pa(base);
- __flush_tsb_one_entry(base, vaddr, xl_hugepage_shift, nentries);
- }
-#endif /* CONFIG_HUGETLB_PAGE */
spin_unlock_irqrestore(&mm->context.lock, flags);
}
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
-static unsigned short hv_pgsz_idx_xlhuge;
-static unsigned int hv_pgsz_mask_xlhuge;
-
-void __init hv_establish_xl_hugepage_tsb_descriptor(unsigned short pgsz_idx,
- unsigned int pgsz_mask)
-{
- hv_pgsz_idx_xlhuge = pgsz_idx;
- hv_pgsz_mask_xlhuge = pgsz_mask;
-}
#endif
-static void sun4v_fill_tsb_descriptor(struct mm_struct *mm)
-{
- struct hv_tsb_descr *htd = &mm->context.tsb_descr[0];
- unsigned int tsb_idx;
-
- for (tsb_idx = 0U; tsb_idx < MM_NUM_TSBS; tsb_idx++) {
- /* Should there not be a tsb then skip it.*/
- if (!mm->context.tsb_block[tsb_idx].tsb)
- continue;
-
- switch (tsb_idx) {
- case MM_TSB_BASE:
- htd->pgsz_mask = HV_PGSZ_MASK_BASE;
- htd->pgsz_idx = HV_PGSZ_IDX_BASE;
- break;
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- case MM_TSB_HUGE:
- htd->pgsz_mask = HV_PGSZ_MASK_HUGE;
- htd->pgsz_idx = HV_PGSZ_IDX_HUGE;
- break;
- case MM_TSB_XLHUGE:
- htd->pgsz_mask = hv_pgsz_mask_xlhuge;
- htd->pgsz_idx = hv_pgsz_idx_xlhuge;
- break;
-#endif
- default:
- BUG();
- }
- htd->assoc = 1;
- htd->num_ttes = mm->context.tsb_block[tsb_idx].tsb_nentries;
- htd->ctx_idx = 0;
- htd->tsb_base = __pa(mm->context.tsb_block[tsb_idx].tsb);
- htd->resv = 0;
- htd++;
- }
-}
-
-static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx,
- unsigned long tsb_bytes)
+static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes)
{
unsigned long tsb_reg, base, tsb_paddr;
unsigned long page_sz, tte;
break;
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
case MM_TSB_HUGE:
- case MM_TSB_XLHUGE:
base = TSBMAP_4M_BASE;
break;
#endif
mm->context.tsb_block[tsb_idx].tsb_map_pte = tte;
}
- if (tlb_type == hypervisor)
- sun4v_fill_tsb_descriptor(mm);
+ /* Setup the Hypervisor TSB descriptor. */
+ if (tlb_type == hypervisor) {
+ struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx];
+
+ switch (tsb_idx) {
+ case MM_TSB_BASE:
+ hp->pgsz_idx = HV_PGSZ_IDX_BASE;
+ break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ case MM_TSB_HUGE:
+ hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
+ break;
+#endif
+ default:
+ BUG();
+ }
+ hp->assoc = 1;
+ hp->num_ttes = tsb_bytes / 16;
+ hp->ctx_idx = 0;
+ switch (tsb_idx) {
+ case MM_TSB_BASE:
+ hp->pgsz_mask = HV_PGSZ_MASK_BASE;
+ break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ case MM_TSB_HUGE:
+ hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
+ break;
+#endif
+ default:
+ BUG();
+ }
+ hp->tsb_base = tsb_paddr;
+ hp->resv = 0;
+ }
}
struct kmem_cache *pgtable_cache __read_mostly;
}
}
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static void capture_and_clear_huge_pte_counts(mm_context_t *mm_context,
- unsigned long *thp_pte_count,
- unsigned long *capture_array)
-{
- unsigned int hugepage_idx;
-
- *thp_pte_count = mm_context->thp_pte_count;
- mm_context->thp_pte_count = 0UL;
-
- for (hugepage_idx = 0UL; hugepage_idx != MM_NUM_HUGEPAGE_SIZES;
- hugepage_idx++) {
- capture_array[hugepage_idx] =
- mm_context->huge_pte_count[hugepage_idx];
- mm_context->huge_pte_count[hugepage_idx] = 0UL;
- }
-}
-
-static void
-captured_hugepage_pte_count_grow_tsb(struct mm_struct *mm,
- unsigned long *thp_pte_count,
- unsigned long *capture_huge_pte_count)
-{
- if (unlikely(capture_huge_pte_count[MM_PTES_HUGE]))
- if (unlikely(capture_huge_pte_count[MM_PTES_HUGE]) || *thp_pte_count)
- tsb_grow(mm, MM_TSB_HUGE,
- (capture_huge_pte_count[MM_PTES_HUGE] +
- *thp_pte_count) * REAL_HPAGE_PER_HPAGE);
-
- if (unlikely(capture_huge_pte_count[MM_PTES_XLHUGE]))
- tsb_grow(mm, MM_TSB_XLHUGE,
- capture_huge_pte_count[MM_PTES_XLHUGE]);
-}
-#else
-static void capture_and_clear_huge_pte_counts(mm_context_t *mm_context,
- unsigned long *thp_pte_count,
- unsigned long *capture_array) {}
-static void
-captured_hugepage_pte_count_grow_tsb(struct mm_struct *mm,
- unsigned long *thp_pte_count,
- unsigned long *capture_huge_pte_count) {}
-#endif /* CONFIG_HUGETLB_PAGE || CONFIG_TRANSPARENT_HUGEPAGE */
-
static atomic_t nctxs = ATOMIC_INIT(0);
int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
{
- unsigned long capture_huge_pte_count[MM_NUM_HUGEPAGE_SIZES];
+ unsigned long mm_rss = get_mm_rss(mm);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ unsigned long saved_hugetlb_pte_count;
unsigned long saved_thp_pte_count;
+#endif
unsigned int i;
int max_nctx = max_user_nctx;
int ret = 0;
mm->context.sparc64_ctx_val = 0UL;
- /* We reset it to zero because the fork() page copying
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ /* We reset them to zero because the fork() page copying
* will re-increment the counters as the parent PTEs are
* copied into the child address space.
*/
- capture_and_clear_huge_pte_counts(&mm->context, &saved_thp_pte_count,
- capture_huge_pte_count);
+ saved_hugetlb_pte_count = mm->context.hugetlb_pte_count;
+ saved_thp_pte_count = mm->context.thp_pte_count;
+ mm->context.hugetlb_pte_count = 0;
+ mm->context.thp_pte_count = 0;
+
+ mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE);
+#endif
/* copy_mm() copies over the parent's mm_struct before calling
* us, so we need to zero out the TSB pointer or else tsb_grow()
/* If this is fork, inherit the parent's TSB size. We would
* grow it to that size on the first page fault anyways.
*/
- tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm) -
- saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
+ tsb_grow(mm, MM_TSB_BASE, mm_rss);
- captured_hugepage_pte_count_grow_tsb(mm, &saved_thp_pte_count,
- capture_huge_pte_count);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count))
+ tsb_grow(mm, MM_TSB_HUGE,
+ (saved_hugetlb_pte_count + saved_thp_pte_count) *
+ REAL_HPAGE_PER_HPAGE);
+#endif
if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) {
ret = -ENOMEM;
projects / users / jedix / linux-maple.git / commitdiff