mm/mmu_notifier: avoid call to invalidate_range() in range_end()

This is an optimization patch that only affect mmu_notifier users which
rely on the invalidate_range() callback.  This patch avoids calling that
callback twice in a row from inside __mmu_notifier_invalidate_range_end

Existing pattern (before this patch):
    mmu_notifier_invalidate_range_start()
        pte/pmd/pud_clear_flush_notify()
            mmu_notifier_invalidate_range()
    mmu_notifier_invalidate_range_end()
        mmu_notifier_invalidate_range()

New pattern (after this patch):
    mmu_notifier_invalidate_range_start()
        pte/pmd/pud_clear_flush_notify()
            mmu_notifier_invalidate_range()
    mmu_notifier_invalidate_range_only_end()

We call the invalidate_range callback after clearing the page table
under the page table lock and we skip the call to invalidate_range
inside the __mmu_notifier_invalidate_range_end() function.

Idea from Andrea Arcangeli

Link: http://lkml.kernel.org/r/20171017031003.7481-3-jglisse@redhat.com
Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Alistair Popple <alistair@popple.id.au>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
Cc: Nadav Amit <nadav.amit@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 130831718e95d736d0e4f8b73bf2e3b32d7f76de..b25dc9db19fc3aee1ccf5fdb0e9a285e77d97b12 100644 (file)
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -214,7 +214,8 @@ extern void __mmu_notifier_change_pte(struct mm_struct *mm,
 extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
                                  unsigned long start, unsigned long end);
 extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-                                 unsigned long start, unsigned long end);
+                                 unsigned long start, unsigned long end,
+                                 bool only_end);
 extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
                                  unsigned long start, unsigned long end);
 
@@ -268,7 +269,14 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
                                  unsigned long start, unsigned long end)
 {
        if (mm_has_notifiers(mm))
-               __mmu_notifier_invalidate_range_end(mm, start, end);
+               __mmu_notifier_invalidate_range_end(mm, start, end, false);
+}
+
+static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
+                                 unsigned long start, unsigned long end)
+{
+       if (mm_has_notifiers(mm))
+               __mmu_notifier_invalidate_range_end(mm, start, end, true);
 }
 
 static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
@@ -439,6 +447,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 {
 }
 
+static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
+                                 unsigned long start, unsigned long end)
+{
+}
+
 static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
                                  unsigned long start, unsigned long end)
 {

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 07ae73f4ef9149eb6b71b1ed9aca9b31f5a9d185..cc65fb87c9db0afd9b6a5b314ae5c3968b9bbf25 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1223,7 +1223,12 @@ static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
        page_remove_rmap(page, true);
        spin_unlock(vmf->ptl);
 
-       mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above pmdp_huge_clear_flush_notify() did already call it.
+        */
+       mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
+                                               mmun_end);
 
        ret |= VM_FAULT_WRITE;
        put_page(page);
@@ -1372,7 +1377,12 @@ alloc:
        }
        spin_unlock(vmf->ptl);
 out_mn:
-       mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above pmdp_huge_clear_flush_notify() did already call it.
+        */
+       mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
+                                              mmun_end);
 out:
        return ret;
 out_unlock:
@@ -2024,7 +2034,12 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
 
 out:
        spin_unlock(ptl);
-       mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PUD_SIZE);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above pudp_huge_clear_flush_notify() did already call it.
+        */
+       mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
+                                              HPAGE_PUD_SIZE);
 }
 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
 
@@ -2099,6 +2114,15 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
                add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
                return;
        } else if (is_huge_zero_pmd(*pmd)) {
+               /*
+                * FIXME: Do we want to invalidate secondary mmu by calling
+                * mmu_notifier_invalidate_range() see comments below inside
+                * __split_huge_pmd() ?
+                *
+                * We are going from a zero huge page write protected to zero
+                * small page also write protected so it does not seems useful
+                * to invalidate secondary mmu at this time.
+                */
                return __split_huge_zero_page_pmd(vma, haddr, pmd);
        }
 
@@ -2234,7 +2258,21 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
        __split_huge_pmd_locked(vma, pmd, haddr, freeze);
 out:
        spin_unlock(ptl);
-       mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback.
+        * They are 3 cases to consider inside __split_huge_pmd_locked():
+        *  1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious
+        *  2) __split_huge_zero_page_pmd() read only zero page and any write
+        *    fault will trigger a flush_notify before pointing to a new page
+        *    (it is fine if the secondary mmu keeps pointing to the old zero
+        *    page in the meantime)
+        *  3) Split a huge pmd into pte pointing to the same page. No need
+        *     to invalidate secondary tlb entry they are all still valid.
+        *     any further changes to individual pte will notify. So no need
+        *     to call mmu_notifier->invalidate_range()
+        */
+       mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
+                                              HPAGE_PMD_SIZE);
 }
 
 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,

diff --git a/mm/memory.c b/mm/memory.c
index a4518aedf4dd133dd227a1c827fd9125c6f44b10..42fb30300bb51a0a79942df7df3c04271e41e03b 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2554,7 +2554,11 @@ static int wp_page_copy(struct vm_fault *vmf)
                put_page(new_page);
 
        pte_unmap_unlock(vmf->pte, vmf->ptl);
-       mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above ptep_clear_flush_notify() did already call it.
+        */
+       mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
        if (old_page) {
                /*
                 * Don't let another task, with possibly unlocked vma,

diff --git a/mm/migrate.c b/mm/migrate.c
index 1236449b4777be3cf61e8cd68616c2c3faa5fd9f..4d0be47a322a8a33491c94072100ebcec829becd 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2089,7 +2089,11 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
        set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
 
        spin_unlock(ptl);
-       mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above pmdp_huge_clear_flush_notify() did already call it.
+        */
+       mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
 
        /* Take an "isolate" reference and put new page on the LRU. */
        get_page(new_page);
@@ -2805,9 +2809,14 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
                        migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
        }
 
+       /*
+        * No need to double call mmu_notifier->invalidate_range() callback as
+        * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
+        * did already call it.
+        */
        if (notified)
-               mmu_notifier_invalidate_range_end(mm, mmu_start,
-                                                 migrate->end);
+               mmu_notifier_invalidate_range_only_end(mm, mmu_start,
+                                                      migrate->end);
 }
 
 /*

diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 314285284e6e6a4c764d0f8126dd0fecf8f7f84e..96edb33fd09a79601ac0a54378ca4ff26df9112a 100644 (file)
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -190,7 +190,9 @@ void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
 
 void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-                                 unsigned long start, unsigned long end)
+                                        unsigned long start,
+                                        unsigned long end,
+                                        bool only_end)
 {
        struct mmu_notifier *mn;
        int id;
@@ -204,8 +206,13 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
                 * subsystem registers either invalidate_range_start()/end() or
                 * invalidate_range(), so this will be no additional overhead
                 * (besides the pointer check).
+                *
+                * We skip call to invalidate_range() if we know it is safe ie
+                * call site use mmu_notifier_invalidate_range_only_end() which
+                * is safe to do when we know that a call to invalidate_range()
+                * already happen under page table lock.
                 */
-               if (mn->ops->invalidate_range)
+               if (!only_end && mn->ops->invalidate_range)
                        mn->ops->invalidate_range(mn, mm, start, end);
                if (mn->ops->invalidate_range_end)
                        mn->ops->invalidate_range_end(mn, mm, start, end);