sparc64 mm: Fix base TSB sizing when hugetlb pages are used

do_sparc64_fault() calculates both the base and huge page RSS sizes and
uses this information in calls to tsb_grow().  The calculation for base
page TSB size is not correct if the task uses hugetlb pages.  hugetlb
pages are not accounted for in RSS, therefore the call to get_mm_rss(mm)
does not include hugetlb pages.  However, the number of pages based on
huge_pte_count (which does include hugetlb pages) is subtracted from
this value.  This will result in an artificially small and often negative
RSS calculation.  The base TSB size is then often set to max_tsb_size
as the passed RSS is unsigned, so a negative value looks really big.

THP pages are also accounted for in huge_pte_count, and THP pages are
accounted for in RSS so the calculation in do_sparc64_fault() is correct
if a task only uses THP pages.

A single huge_pte_count is not sufficient for TSB sizing if both hugetlb
and THP pages can be used.  Instead of a single counter, use two:  one
for hugetlb and one for THP.

Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h
index 70067ce184b16a9d91de2737bf804dbb89e94307..f7de0dbc38af2dd36c9f34df53e6e951f6729825 100644 (file)
--- a/arch/sparc/include/asm/mmu_64.h
+++ b/arch/sparc/include/asm/mmu_64.h
@@ -92,7 +92,8 @@ struct tsb_config {
 typedef struct {
        spinlock_t              lock;
        unsigned long           sparc64_ctx_val;
-       unsigned long           huge_pte_count;
+       unsigned long           hugetlb_pte_count;
+       unsigned long           thp_pte_count;
        struct tsb_config       tsb_block[MM_NUM_TSBS];
        struct hv_tsb_descr     tsb_descr[MM_NUM_TSBS];
 } mm_context_t;

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 6c43b924a7a2bae17cfd98b4b7de0be64b16ad13..3a16ba0dc356d2a7230febdfef3801659a269d67 100644 (file)
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -476,14 +476,14 @@ good_area:
        up_read(&mm->mmap_sem);
 
        mm_rss = get_mm_rss(mm);
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
+       mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
 #endif
        if (unlikely(mm_rss >
                     mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
                tsb_grow(mm, MM_TSB_BASE, mm_rss);
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       mm_rss = mm->context.huge_pte_count;
+       mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
        if (unlikely(mm_rss >
                     mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
                if (mm->context.tsb_block[MM_TSB_HUGE].tsb)

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index ba52e6466a8252659d0f19e49abb6cba1a7762b2..d69b66e01b843dcff392b85609e840acd827bb27 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -180,7 +180,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
        unsigned long nptes;
 
        if (!pte_present(*ptep) && pte_present(entry))
-               mm->context.huge_pte_count++;
+               mm->context.hugetlb_pte_count++;
 
        addr &= HPAGE_MASK;
 
@@ -212,7 +212,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 
        entry = *ptep;
        if (pte_present(entry))
-               mm->context.huge_pte_count--;
+               mm->context.hugetlb_pte_count--;
 
        addr &= HPAGE_MASK;
        nptes = 1 << HUGETLB_PAGE_ORDER;

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index aec508e374906ba0b1860f3669a609407c080125..aef153f9fdac21575e3cbfb8b089f779b7e73117 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -346,7 +346,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
        spin_lock_irqsave(&mm->context.lock, flags);
 
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
+       if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) &&
+           is_hugetlb_pte(pte))
                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
                                        address, pte_val(pte));
        else

diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index f81cd973670079132681d69d0b9800bb12bfbb55..3659d37b4d818e30c614f46cf4b2aca8bf700aa0 100644 (file)
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -175,9 +175,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 
        if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
                if (pmd_val(pmd) & _PAGE_PMD_HUGE)
-                       mm->context.huge_pte_count++;
+                       mm->context.thp_pte_count++;
                else
-                       mm->context.huge_pte_count--;
+                       mm->context.thp_pte_count--;
 
                /* Do not try to allocate the TSB hash table if we
                 * don't have one already.  We have various locks held

diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index a0604a493a361e51c055e825e65295a302aeec4c..6725ed45580e525cf5567b1b9ccbeb2c723738bf 100644 (file)
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -470,7 +470,7 @@ retry_tsb_alloc:
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       unsigned long huge_pte_count;
+       unsigned long total_huge_pte_count;
 #endif
        unsigned int i;
 
@@ -479,12 +479,14 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
        mm->context.sparc64_ctx_val = 0UL;
 
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       /* We reset it to zero because the fork() page copying
+       /* We reset them to zero because the fork() page copying
         * will re-increment the counters as the parent PTEs are
         * copied into the child address space.
         */
-       huge_pte_count = mm->context.huge_pte_count;
-       mm->context.huge_pte_count = 0;
+       total_huge_pte_count = mm->context.hugetlb_pte_count +
+                        mm->context.thp_pte_count;
+       mm->context.hugetlb_pte_count = 0;
+       mm->context.thp_pte_count = 0;
 #endif
 
        /* copy_mm() copies over the parent's mm_struct before calling
@@ -500,8 +502,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
        tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
 
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-       if (unlikely(huge_pte_count))
-               tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
+       if (unlikely(total_huge_pte_count))
+               tsb_grow(mm, MM_TSB_HUGE, total_huge_pte_count);
 #endif
 
        if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))