sparc64: Reduce TLB flushes during hugepte changes

During hugepage map/unmap, TSB and TLB flushes are currently
issued at every PAGE_SIZE'd boundary which is unnecessary.
We now issue the flush at REAL_HPAGE_SIZE boundaries only.

Without this patch workloads which unmap a large hugepage
backed VMA region get CPU lockups due to excessive TLB
flush calls.

Orabug: 23071722

Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
(cherry picked from commit b42a694198cca38e8cdb3f601266bf591ba3291d)
(cherry picked from commit fdc7f39ae632a9ec0114c59090131d2db7dd7682)
Signed-off-by: Allen Pais <allen.pais@oracle.com>

diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 9217c9f89eb3ee4adbd70453f3163d9a7eb91e57..ed7a88bcd7fd5c16312ab809e86e4b5542c753d6 100644 (file)
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -381,7 +381,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot)
 extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
                                struct page *page, int writable);
 #define        arch_make_huge_pte arch_make_huge_pte
-static inline pte_t pte_mkhuge(pte_t pte)
+static inline unsigned long __pte_default_huge_mask(void)
 {
        unsigned long mask;
 
@@ -396,8 +396,21 @@ static inline pte_t pte_mkhuge(pte_t pte)
        : "=r" (mask)
        : "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
 
-       return __pte(pte_val(pte) | mask);
+       return mask;
+}
+
+static inline pte_t pte_mkhuge(pte_t pte)
+{
+       return __pte(pte_val(pte) | __pte_default_huge_mask());
+}
+
+static inline bool is_default_hugetlb_pte(pte_t pte)
+{
+       unsigned long mask = __pte_default_huge_mask();
+
+       return (pte_val(pte) & mask) == mask;
 }
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline pmd_t pmd_mkhuge(pmd_t pmd)
 {
@@ -409,6 +422,11 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
        return __pmd(pte_val(pte));
 }
 #endif
+#else
+static inline bool is_hugetlb_pte(pte_t pte)
+{
+       return false;
+}
 #endif
 
 static inline pte_t pte_mkdirty(pte_t pte)
@@ -871,6 +889,19 @@ static inline unsigned long pud_pfn(pud_t pud)
 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
                   pte_t *ptep, pte_t orig, int fullmm);
 
+static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
+                               pte_t *ptep, pte_t orig, int fullmm)
+{
+       /* It is more efficient to let flush_tlb_kernel_range()
+        * handle init_mm tlb flushes.
+        *
+        * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
+        *             and SUN4V pte layout, so this inline test is fine.
+        */
+       if (likely(mm != &init_mm) && pte_accessible(mm, orig))
+               tlb_batch_add(mm, vaddr, ptep, orig, fullmm);
+}
+
 #define __HAVE_ARCH_PMDP_GET_AND_CLEAR
 static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
                                       unsigned long addr,
@@ -887,15 +918,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
        pte_t orig = *ptep;
 
        *ptep = pte;
-
-       /* It is more efficient to let flush_tlb_kernel_range()
-        * handle init_mm tlb flushes.
-        *
-        * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
-        *             and SUN4V pte layout, so this inline test is fine.
-        */
-       if (likely(mm != &init_mm) && pte_accessible(mm, orig))
-               tlb_batch_add(mm, addr, ptep, orig, fullmm);
+       maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm);
 }
 
 #define set_pte_at(mm,addr,ptep,pte)   \

diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h
index dea1cfa2122bec28ea36a34bec543d2dc4747f9b..8c8e24fedfb8ef5facce74505193ea12627461c8 100644 (file)
--- a/arch/sparc/include/asm/tlbflush_64.h
+++ b/arch/sparc/include/asm/tlbflush_64.h
@@ -8,6 +8,7 @@
 #define TLB_BATCH_NR   192
 
 struct tlb_batch {
+       bool default_huge;
        struct mm_struct *mm;
        unsigned long tlb_nr;
        unsigned long active;
@@ -16,7 +17,7 @@ struct tlb_batch {
 
 void flush_tsb_kernel_range(unsigned long start, unsigned long end);
 void flush_tsb_user(struct tlb_batch *tb);
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr);
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge);
 
 /* TLB flush operations. */
 

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 86bb203af1dc0ef86d2908c0460fecfc499ee829..298fa1e6a7b72d26e1a84074aaf66421e8227c31 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -336,9 +336,9 @@ static bool huge_pte_at_handle_sentinel(pte_t *sentinel_pte, pte_t *pte,
 }
 
 static bool __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-                             pte_t *pte, pte_t entry, pte_t *sentinel_pte)
+                             pte_t *pte, pte_t entry, pte_t *sentinel_pte,
+                             unsigned int hugepage_shift)
 {
-       unsigned int hugepage_shift = tte_to_shift(entry);
        bool rc = true;
 
        if (hugepage_shift != REAL_HPAGE_SHIFT) {
@@ -348,23 +348,22 @@ static bool __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
                                        entry);
                huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte);
        } else
-               set_pte_at(mm, addr, pte, entry);
+               *pte = entry;
 
        return rc;
 }
 
 static void __clear_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-                               pte_t *pte, pte_t *sentinel_pte)
+                               pte_t *pte, pte_t *sentinel_pte,
+                               unsigned int hugepage_shift)
 {
-       unsigned int hugepage_shift = tte_to_shift(*pte);
-
        if (hugepage_shift != REAL_HPAGE_SHIFT) {
                pte_t orig = *pte;
 
                *pte = __pte(0UL);
                huge_pte_at_flush_update(mm, addr, pte, orig, sentinel_pte);
        } else
-               pte_clear(mm, addr, pte);
+               *pte = __pte(0UL);
 }
 
 static bool set_huge_pte_range_at(struct mm_struct *mm, pmd_t *pmd,
@@ -374,20 +373,37 @@ static bool set_huge_pte_range_at(struct mm_struct *mm, pmd_t *pmd,
        pte_t *pte = pte_offset_map(pmd, addr);
        pte_t *lpte = pte + PTRS_PER_PTE;
        pte_t entry = *pentry;
+       pte_t orig = *(pte_t *)pte;
        bool rc = true;
+       unsigned long orig_addr = addr;
+       unsigned int hugepage_shift;
+
+       if (set_at)
+               hugepage_shift = tte_to_shift(entry);
+       else
+               hugepage_shift = tte_to_shift(*pte);
 
        for (; pte < lpte; pte++, addr = addr + PAGE_SIZE) {
                if (set_at) {
                        rc = __set_huge_pte_at(mm, addr, pte, entry,
-                                                       sentinel_pte);
+                                       sentinel_pte, hugepage_shift);
                        if (!rc)
                                break;
                        pte_val(entry) = pte_val(entry) + PAGE_SIZE;
                } else
-                       __clear_huge_pte_at(mm, addr, pte, sentinel_pte);
+                       __clear_huge_pte_at(mm, addr, pte, sentinel_pte,
+                                       hugepage_shift);
        }
        if (set_at)
                *pentry = entry;
+
+       if (hugepage_shift == REAL_HPAGE_SHIFT) {
+               /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
+               maybe_tlb_batch_add(mm, orig_addr, pte, orig, 0);
+               maybe_tlb_batch_add(mm, orig_addr + REAL_HPAGE_SIZE,
+                                       pte, orig, 0);
+       }
+
        return rc;
 }
 

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 473f2fef6fbb718150514d5023c3fdaa83f33dc0..12b3cafd0f4f9b588dee377d06e178cdeff1b3cc 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -437,18 +437,6 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde
        tsb_insert(tsb, tag, tte);
 }
 
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline bool is_hugetlb_pte(pte_t pte)
-{
-       if ((tlb_type == hypervisor &&
-            (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
-           (tlb_type != hypervisor &&
-            (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))
-               return true;
-       return false;
-}
-#endif
-
 #ifdef CONFIG_HUGETLB_PAGE
 unsigned int xl_hugepage_shift;
 static unsigned long xl_hugepage_pte;
@@ -618,7 +606,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
        spin_lock_irqsave(&mm->context.lock, flags);
 
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       if (mm->context.huge_pte_count[MM_PTES_HUGE] && is_hugetlb_pte(pte))
+       if (mm->context.huge_pte_count[MM_PTES_HUGE] &&
+                       is_default_hugetlb_pte(pte))
                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
                                        address, pte_val(pte));
        else if (mm->context.huge_pte_count[MM_PTES_XLHUGE] &&

diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 6e6633094557185f3277137cc2ba419aebdc1f23..5b0c4738296ece308db701da5f5dfed02f2b1762 100644 (file)
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void)
 }
 
 static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
-                             bool exec)
+                             bool exec, bool default_huge)
 {
        struct tlb_batch *tb = &get_cpu_var(tlb_batch);
        unsigned long nr;
@@ -84,13 +84,21 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
        }
 
        if (!tb->active) {
-               flush_tsb_user_page(mm, vaddr);
+               flush_tsb_user_page(mm, vaddr, default_huge);
                global_flush_tlb_page(mm, vaddr);
                goto out;
        }
 
-       if (nr == 0)
+       if (nr == 0) {
                tb->mm = mm;
+               tb->default_huge = default_huge;
+       }
+
+       if (tb->default_huge != default_huge) {
+               flush_tlb_pending();
+               tb->default_huge = default_huge;
+               nr = 0;
+       }
 
        tb->vaddrs[nr] = vaddr;
        tb->tlb_nr = ++nr;
@@ -104,6 +112,8 @@ out:
 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
                   pte_t *ptep, pte_t orig, int fullmm)
 {
+       bool default_huge = is_default_hugetlb_pte(orig);
+
        if (tlb_type != hypervisor &&
            pte_dirty(orig)) {
                unsigned long paddr, pfn = pte_pfn(orig);
@@ -129,7 +139,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
 
 no_cache_flush:
        if (!fullmm)
-               tlb_batch_add_one(mm, vaddr, pte_exec(orig));
+               tlb_batch_add_one(mm, vaddr, pte_exec(orig), default_huge);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -145,7 +155,7 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
                if (pte_val(*pte) & _PAGE_VALID) {
                        bool exec = pte_exec(*pte);
 
-                       tlb_batch_add_one(mm, vaddr, exec);
+                       tlb_batch_add_one(mm, vaddr, exec, false);
                }
                pte++;
                vaddr += PAGE_SIZE;
@@ -185,8 +195,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                        pte_t orig_pte = __pte(pmd_val(orig));
                        bool exec = pte_exec(orig_pte);
 
-                       tlb_batch_add_one(mm, addr, exec);
-                       tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
+                       tlb_batch_add_one(mm, addr, exec, true);
+                       tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
+                                       true);
                } else {
                        tlb_batch_pmd_scan(mm, addr, orig);
                }

diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 0bbf88445b8da89faaf5e695cecf340cf9703689..19087ffed9177e0a5298a545381df77155349267 100644 (file)
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -76,14 +76,15 @@ void flush_tsb_user(struct tlb_batch *tb)
 
        spin_lock_irqsave(&mm->context.lock, flags);
 
-       base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
-       nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
-       if (tlb_type == cheetah_plus || tlb_type == hypervisor)
-               base = __pa(base);
-       __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
-
+       if (!tb->default_huge) {
+               base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+               nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+               if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+                       base = __pa(base);
+               __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
+       }
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+       if (tb->default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
                base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
                nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
                if (tlb_type == cheetah_plus || tlb_type == hypervisor)
@@ -92,7 +93,7 @@ void flush_tsb_user(struct tlb_batch *tb)
        }
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
-       if (mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
+       if (!tb->default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
                base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb;
                nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries;
                base = __pa(base);
@@ -102,20 +103,22 @@ void flush_tsb_user(struct tlb_batch *tb)
        spin_unlock_irqrestore(&mm->context.lock, flags);
 }
 
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr,
+                       bool default_huge)
 {
        unsigned long nentries, base, flags;
 
        spin_lock_irqsave(&mm->context.lock, flags);
 
-       base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
-       nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
-       if (tlb_type == cheetah_plus || tlb_type == hypervisor)
-               base = __pa(base);
-       __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
-
+       if (!default_huge) {
+               base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+               nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+               if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+                       base = __pa(base);
+               __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
+       }
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+       if (default_huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
                base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
                nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
                if (tlb_type == cheetah_plus || tlb_type == hypervisor)
@@ -124,7 +127,7 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
        }
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
-       if (mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
+       if (!default_huge && mm->context.tsb_block[MM_TSB_XLHUGE].tsb) {
                base = (unsigned long) mm->context.tsb_block[MM_TSB_XLHUGE].tsb;
                nentries = mm->context.tsb_block[MM_TSB_XLHUGE].tsb_nentries;
                base = __pa(base);