mm, thp: avoid unlikely branches for split_huge_pmd

While doing MADV_DONTNEED on a large area of thp memory, I noticed we
encountered many unlikely() branches in profiles for each backing
hugepage.  This is because zap_pmd_range() would call split_huge_pmd(),
which rechecked the conditions that were already validated, but as part
of an unlikely() branch.

Avoid the unlikely() branch when in a context where pmd is known to be
good for __split_huge_pmd() directly.

Link: http://lkml.kernel.org/r/alpine.DEB.2.10.1610181600300.84525@chino.kir.corp.google.com
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index e35e6de633b9a7bc2a080d6b3596aa16e8c8582f..1f782aa1d8e60c286ae99a50bfc0819d55ea890a 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -189,6 +189,8 @@ static inline void deferred_split_huge_page(struct page *page) {}
 #define split_huge_pmd(__vma, __pmd, __address)        \
        do { } while (0)
 
+static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
+               unsigned long address, bool freeze, struct page *page) {}
 static inline void split_huge_pmd_address(struct vm_area_struct *vma,
                unsigned long address, bool freeze, struct page *page) {}
 

diff --git a/mm/memory.c b/mm/memory.c
index 33f45edf82721f5977ed732e4f07db447ecc9a92..d86b7b4afd7d4a13874911eb4ee2b5a942ad0044 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1240,7 +1240,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                        if (next - addr != HPAGE_PMD_SIZE) {
                                VM_BUG_ON_VMA(vma_is_anonymous(vma) &&
                                    !rwsem_is_locked(&tlb->mm->mmap_sem), vma);
-                               split_huge_pmd(vma, pmd, addr);
+                               __split_huge_pmd(vma, pmd, addr, false, NULL);
                        } else if (zap_huge_pmd(tlb, vma, pmd, addr))
                                goto next;
                        /* fall through */
@@ -3454,7 +3454,7 @@ static int wp_huge_pmd(struct fault_env *fe, pmd_t orig_pmd)
 
        /* COW handled on pte level: split pmd */
        VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma);
-       split_huge_pmd(fe->vma, fe->pmd, fe->address);
+       __split_huge_pmd(fe->vma, fe->pmd, fe->address, false, NULL);
 
        return VM_FAULT_FALLBACK;
 }

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 0b859af06b87df4e17af4180b953a6337c251dff..a6a27e5d6b141da21d528c2e7400f0891d52c6a3 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -496,7 +496,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
                        page = pmd_page(*pmd);
                        if (is_huge_zero_page(page)) {
                                spin_unlock(ptl);
-                               split_huge_pmd(vma, pmd, addr);
+                               __split_huge_pmd(vma, pmd, addr, false, NULL);
                        } else {
                                get_page(page);
                                spin_unlock(ptl);

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 05a02b72c98dae137d6d3e8aa2a75e4d9c8ba2f5..c5ba2aae0f542d03d144057ec49ba9a9fdc36456 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -176,7 +176,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
 
                if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
                        if (next - addr != HPAGE_PMD_SIZE) {
-                               split_huge_pmd(vma, pmd, addr);
+                               __split_huge_pmd(vma, pmd, addr, false, NULL);
                                if (pmd_trans_unstable(pmd))
                                        continue;
                        } else {