mm: remove the page size change check in tlb_remove_page

Now that we check for page size change early in the loop, we can
partially revert e9d55e157034a ("mm: change the interface for
__tlb_remove_page").

This simplies the code much, by removing the need to track the last
address with which we adjusted the range.  We also go back to the older
way of filling the mmu_gather array, ie, we add an entry and then check
whether the gather batch is full.

Link: http://lkml.kernel.org/r/20161026084839.27299-6-aneesh.kumar@linux.vnet.ibm.com
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index a9d6de4746ea762ec912e406d322228a97220bbf..3f2eb76243e3c5f9d387959acae740ce871e5afa 100644 (file)
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -213,18 +213,17 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 
 static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
+       tlb->pages[tlb->nr++] = page;
+       VM_WARN_ON(tlb->nr > tlb->max);
        if (tlb->nr == tlb->max)
                return true;
-       tlb->pages[tlb->nr++] = page;
        return false;
 }
 
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
-       if (__tlb_remove_page(tlb, page)) {
+       if (__tlb_remove_page(tlb, page))
                tlb_flush_mmu(tlb);
-               __tlb_remove_page(tlb, page);
-       }
 }
 
 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
@@ -233,12 +232,6 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
        return __tlb_remove_page(tlb, page);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
-                                        struct page *page)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {

diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h
index bfe6295aa746829d9b2cc0d48d99e75d46d70219..fced197b96264e01b20743e90706ed20cf30b242 100644 (file)
--- a/arch/ia64/include/asm/tlb.h
+++ b/arch/ia64/include/asm/tlb.h
@@ -207,15 +207,15 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
  */
 static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
-       if (tlb->nr == tlb->max)
-               return true;
-
        tlb->need_flush = 1;
 
        if (!tlb->nr && tlb->pages == tlb->local)
                __tlb_alloc_page(tlb);
 
        tlb->pages[tlb->nr++] = page;
+       VM_WARN_ON(tlb->nr > tlb->max);
+       if (tlb->nr == tlb->max)
+               return true;
        return false;
 }
 
@@ -236,10 +236,8 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb)
 
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
-       if (__tlb_remove_page(tlb, page)) {
+       if (__tlb_remove_page(tlb, page))
                tlb_flush_mmu(tlb);
-               __tlb_remove_page(tlb, page);
-       }
 }
 
 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
@@ -248,12 +246,6 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
        return __tlb_remove_page(tlb, page);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
-                                        struct page *page)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {

diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 28b159c87c3873dcfa862b14a4c64d3151e323b5..853b2a3d8deeeac08254be515502500a7842a3d9 100644 (file)
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -104,12 +104,6 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
        return __tlb_remove_page(tlb, page);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
-                                        struct page *page)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {

diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 0f988b3e484bfe59d3c1e7b46ef98337bc8a4945..46e0d635e36f711aff9a88c45955905d7fbf3cc2 100644 (file)
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -118,12 +118,6 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
        return __tlb_remove_page(tlb, page);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
-                                        struct page *page)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {

diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h
index 8258dd4bb13c3791f3bf8a64b7832f6a019beb90..600a2e9bfee2feea2a6dbc8b91d2a5a872d9d8d3 100644 (file)
--- a/arch/um/include/asm/tlb.h
+++ b/arch/um/include/asm/tlb.h
@@ -116,12 +116,6 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
        return __tlb_remove_page(tlb, page);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
-                                        struct page *page)
-{
-       return __tlb_remove_page(tlb, page);
-}
-
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {

diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 256c9de71fdba7b0558e21df43935db73c6ebe8a..7eed8cf3130ada017b658649f543aba980085da1 100644 (file)
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -107,11 +107,6 @@ struct mmu_gather {
        struct mmu_gather_batch local;
        struct page             *__pages[MMU_GATHER_BUNDLE];
        unsigned int            batch_count;
-       /*
-        * __tlb_adjust_range  will track the new addr here,
-        * that that we can adjust the range after the flush
-        */
-       unsigned long addr;
        int page_size;
 };
 
@@ -130,12 +125,6 @@ static inline void __tlb_adjust_range(struct mmu_gather *tlb,
 {
        tlb->start = min(tlb->start, address);
        tlb->end = max(tlb->end, address + range_size);
-       /*
-        * Track the last address with which we adjusted the range. This
-        * will be used later to adjust again after a mmu_flush due to
-        * failed __tlb_remove_page
-        */
-       tlb->addr = address;
 }
 
 static inline void __tlb_reset_range(struct mmu_gather *tlb)
@@ -151,15 +140,11 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb)
 static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
 {
-       if (__tlb_remove_page_size(tlb, page, page_size)) {
+       if (__tlb_remove_page_size(tlb, page, page_size))
                tlb_flush_mmu(tlb);
-               tlb->page_size = page_size;
-               __tlb_adjust_range(tlb, tlb->addr, page_size);
-               __tlb_remove_page_size(tlb, page, page_size);
-       }
 }
 
-static bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
        return __tlb_remove_page_size(tlb, page, PAGE_SIZE);
 }
@@ -173,15 +158,6 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        return tlb_remove_page_size(tlb, page, PAGE_SIZE);
 }
 
-static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page)
-{
-       /* active->nr should be zero when we call this */
-       VM_BUG_ON_PAGE(tlb->active->nr, page);
-       tlb->page_size = PAGE_SIZE;
-       __tlb_adjust_range(tlb, tlb->addr, PAGE_SIZE);
-       return __tlb_remove_page(tlb, page);
-}
-
 #ifndef tlb_remove_check_page_size_change
 #define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
 static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,

diff --git a/mm/memory.c b/mm/memory.c
index eae20eb66bfc454d7d473290afe70d65fd6f249b..0a72f821ccdcc859e80834fa5bf0b005914001b3 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -300,15 +300,14 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_
        struct mmu_gather_batch *batch;
 
        VM_BUG_ON(!tlb->end);
-
-       if (!tlb->page_size)
-               tlb->page_size = page_size;
-       else {
-               if (page_size != tlb->page_size)
-                       return true;
-       }
+       VM_WARN_ON(tlb->page_size != page_size);
 
        batch = tlb->active;
+       /*
+        * Add the page and check if we are full. If so
+        * force a flush.
+        */
+       batch->pages[batch->nr++] = page;
        if (batch->nr == batch->max) {
                if (!tlb_next_batch(tlb))
                        return true;
@@ -316,7 +315,6 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_
        }
        VM_BUG_ON_PAGE(batch->nr > batch->max, page);
 
-       batch->pages[batch->nr++] = page;
        return false;
 }
 
@@ -1122,7 +1120,6 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
        pte_t *start_pte;
        pte_t *pte;
        swp_entry_t entry;
-       struct page *pending_page = NULL;
 
        tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
 again:
@@ -1177,7 +1174,6 @@ again:
                                print_bad_pte(vma, addr, ptent, page);
                        if (unlikely(__tlb_remove_page(tlb, page))) {
                                force_flush = 1;
-                               pending_page = page;
                                addr += PAGE_SIZE;
                                break;
                        }
@@ -1218,11 +1214,6 @@ again:
        if (force_flush) {
                force_flush = 0;
                tlb_flush_mmu_free(tlb);
-               if (pending_page) {
-                       /* remove the page with new size */
-                       __tlb_remove_pte_page(tlb, pending_page);
-                       pending_page = NULL;
-               }
                if (addr != end)
                        goto again;
        }