mm/munlock: delete munlock_vma_pages_all(), allow oomreap

munlock_vma_pages_range() will still be required, when munlocking but
not munmapping a set of pages; but when unmapping a pte, the mlock count
will be maintained in much the same way as it will be maintained when
mapping in the pte.  Which removes the need for munlock_vma_pages_all()
on mlocked vmas when munmapping or exiting: eliminating the catastrophic
contention on i_mmap_rwsem, and the need for page lock on the pages.

There is still a need to update locked_vm accounting according to the
munmapped vmas when munmapping: do that in detach_vmas_to_be_unmapped().
exit_mmap() does not need locked_vm updates, so delete unlock_range().

And wasn't I the one who forbade the OOM reaper to attack mlocked vmas,
because of the uncertainty in blocking on all those page locks?
No fear of that now, so permit the OOM reaper on mlocked vmas.

Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

diff --git a/mm/internal.h b/mm/internal.h
index e48c486d5ddfb5dbd64a6956e2863cb61da32ad1..f235aa92e564773432dcd212f0c61c3277558bf7 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -71,11 +71,6 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
                unsigned long floor, unsigned long ceiling);
 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
 
-static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
-{
-       return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
-}
-
 struct zap_details;
 void unmap_page_range(struct mmu_gather *tlb,
                             struct vm_area_struct *vma,
@@ -398,12 +393,8 @@ extern long populate_vma_page_range(struct vm_area_struct *vma,
 extern long faultin_vma_page_range(struct vm_area_struct *vma,
                                   unsigned long start, unsigned long end,
                                   bool write, int *locked);
-extern void munlock_vma_pages_range(struct vm_area_struct *vma,
-                       unsigned long start, unsigned long end);
-static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
-{
-       munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
-}
+extern int mlock_future_check(struct mm_struct *mm, unsigned long flags,
+                             unsigned long len);
 
 /*
  * must be called with vma's mmap_lock held for read or write, and page locked.
@@ -411,9 +402,6 @@ static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
 extern void mlock_vma_page(struct page *page);
 extern void munlock_vma_page(struct page *page);
 
-extern int mlock_future_check(struct mm_struct *mm, unsigned long flags,
-                             unsigned long len);
-
 /*
  * Clear the page's PageMlocked().  This can be useful in a situation where
  * we want to unconditionally remove a page from the pagecache -- e.g.,

diff --git a/mm/madvise.c b/mm/madvise.c
index 5604064df4646e786d1c9e2fcf0a35eb71f622ba..ae35d72627efab9074288c57bfda5a02880c8396 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -530,6 +530,11 @@ static void madvise_cold_page_range(struct mmu_gather *tlb,
        tlb_end_vma(tlb, vma);
 }
 
+static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
+{
+       return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
+}
+
 static long madvise_cold(struct vm_area_struct *vma,
                        struct vm_area_struct **prev,
                        unsigned long start_addr, unsigned long end_addr)

diff --git a/mm/mlock.c b/mm/mlock.c
index aec4ce7919daef9e2d0f7b3eefee3c48144807d8..5d7ced8303bebe18d235ecc31c43974d38b3e301 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -137,8 +137,8 @@ void munlock_vma_page(struct page *page)
  * Returns with VM_LOCKED cleared.  Callers must be prepared to
  * deal with this.
  */
-void munlock_vma_pages_range(struct vm_area_struct *vma,
-                            unsigned long start, unsigned long end)
+static void munlock_vma_pages_range(struct vm_area_struct *vma,
+                                   unsigned long start, unsigned long end)
 {
        vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
 

diff --git a/mm/mmap.c b/mm/mmap.c
index 1e8fdb0b51eddcb8bce46b6f7c0648980827ea04..64b5985b5295c1abe87daeb6ffe87a031dde042c 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2674,6 +2674,8 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
        vma->vm_prev = NULL;
        do {
                vma_rb_erase(vma, &mm->mm_rb);
+               if (vma->vm_flags & VM_LOCKED)
+                       mm->locked_vm -= vma_pages(vma);
                mm->map_count--;
                tail_vma = vma;
                vma = vma->vm_next;
@@ -2778,22 +2780,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
        return __split_vma(mm, vma, addr, new_below);
 }
 
-static inline void
-unlock_range(struct vm_area_struct *start, unsigned long limit)
-{
-       struct mm_struct *mm = start->vm_mm;
-       struct vm_area_struct *tmp = start;
-
-       while (tmp && tmp->vm_start < limit) {
-               if (tmp->vm_flags & VM_LOCKED) {
-                       mm->locked_vm -= vma_pages(tmp);
-                       munlock_vma_pages_all(tmp);
-               }
-
-               tmp = tmp->vm_next;
-       }
-}
-
 /* Munmap is split into 2 main parts -- this part which finds
  * what needs doing, and the areas themselves, which do the
  * work.  This now handles partial unmappings.
@@ -2874,12 +2860,6 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
                        return error;
        }
 
-       /*
-        * unlock any mlock()ed ranges before detaching vmas
-        */
-       if (mm->locked_vm)
-               unlock_range(vma, end);
-
        /* Detach vmas from rbtree */
        if (!detach_vmas_to_be_unmapped(mm, vma, prev, end))
                downgrade = false;
@@ -3147,20 +3127,12 @@ void exit_mmap(struct mm_struct *mm)
                 * Nothing can be holding mm->mmap_lock here and the above call
                 * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
                 * __oom_reap_task_mm() will not block.
-                *
-                * This needs to be done before calling unlock_range(),
-                * which clears VM_LOCKED, otherwise the oom reaper cannot
-                * reliably test it.
                 */
                (void)__oom_reap_task_mm(mm);
-
                set_bit(MMF_OOM_SKIP, &mm->flags);
        }
 
        mmap_write_lock(mm);
-       if (mm->locked_vm)
-               unlock_range(mm->mmap, ULONG_MAX);
-
        arch_exit_mmap(mm);
 
        vma = mm->mmap;

diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 832fb330376efe9f509045516a72d795af5e39a8..6b875acabd1e7258ca0e1c224f991d46ca8499ed 100644 (file)
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -526,7 +526,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm)
        set_bit(MMF_UNSTABLE, &mm->flags);
 
        for (vma = mm->mmap ; vma; vma = vma->vm_next) {
-               if (!can_madv_lru_vma(vma))
+               if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP))
                        continue;
 
                /*