mm: avoid assertion in untrack_pfn

untrack_pfn can be called after VMA was isolated and mmap_lock downgraded.
An attempt to lock affected VMA would cause an assertion, therefore
use mod_vm_flags_nolock in such situations.

Signed-off-by: Suren Baghdasaryan <surenb@google.com>

diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c
index 926fa2d3d725c3590cf26135af7086ed797cfe96..b5c82a0c552f5d8bcadb9a55a511c01c9239a744 100644 (file)
--- a/arch/x86/mm/pat/memtype.c
+++ b/arch/x86/mm/pat/memtype.c
@@ -1092,7 +1092,7 @@ void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, pfn_t pfn)
  * can be for the entire vma (in which case pfn, size are zero).
  */
 void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
-                unsigned long size)
+                unsigned long size, bool lock_vma)
 {
        resource_size_t paddr;
        unsigned long prot;
@@ -1111,8 +1111,12 @@ void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
                size = vma->vm_end - vma->vm_start;
        }
        free_pfn_range(paddr, size);
-       if (vma)
-               clear_vm_flags(vma, VM_PAT);
+       if (vma) {
+               if (lock_vma)
+                       clear_vm_flags(vma, VM_PAT);
+               else
+                       mod_vm_flags_nolock(vma, 0, VM_PAT);
+       }
 }
 
 /*

diff --git a/include/linux/mm.h b/include/linux/mm.h
index cb2d7f8ae53230aed5db8eacf5d7b0466a2a87c0..17b6aa12c47a3b566dbdbd3f46b050bf3562563e 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2009,7 +2009,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long address,
                    unsigned long size);
 void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt,
                struct vm_area_struct *start_vma, unsigned long start,
-               unsigned long end);
+               unsigned long end, bool lock_vma);
 
 struct mmu_notifier_range;
 

diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index a108b60a6962b343d5569b05388d6e8d5af14254..7bd4274234f330b9255b388adb9026a7a08c9472 100644 (file)
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -1191,7 +1191,8 @@ static inline int track_pfn_copy(struct vm_area_struct *vma)
  * can be for the entire vma (in which case pfn, size are zero).
  */
 static inline void untrack_pfn(struct vm_area_struct *vma,
-                              unsigned long pfn, unsigned long size)
+                              unsigned long pfn, unsigned long size,
+                              bool lock_vma)
 {
 }
 
@@ -1209,7 +1210,7 @@ extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
                             pfn_t pfn);
 extern int track_pfn_copy(struct vm_area_struct *vma);
 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
-                       unsigned long size);
+                       unsigned long size, bool lock_vma);
 extern void untrack_pfn_moved(struct vm_area_struct *vma);
 #endif
 

diff --git a/mm/memory.c b/mm/memory.c
index a4982e16cf19fc0526457d88e6b6257be1b429e0..d07015522a75053bd4b7d00add07a3ee6e7ab87d 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1658,7 +1658,7 @@ void unmap_page_range(struct mmu_gather *tlb,
 static void unmap_single_vma(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr,
-               struct zap_details *details)
+               struct zap_details *details, bool lock_vma)
 {
        unsigned long start = max(vma->vm_start, start_addr);
        unsigned long end;
@@ -1673,7 +1673,7 @@ static void unmap_single_vma(struct mmu_gather *tlb,
                uprobe_munmap(vma, start, end);
 
        if (unlikely(vma->vm_flags & VM_PFNMAP))
-               untrack_pfn(vma, 0, 0);
+               untrack_pfn(vma, 0, 0, lock_vma);
 
        if (start != end) {
                if (unlikely(is_vm_hugetlb_page(vma))) {
@@ -1720,7 +1720,7 @@ static void unmap_single_vma(struct mmu_gather *tlb,
  */
 void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt,
                struct vm_area_struct *vma, unsigned long start_addr,
-               unsigned long end_addr)
+               unsigned long end_addr, bool lock_vma)
 {
        struct mmu_notifier_range range;
        struct zap_details details = {
@@ -1734,7 +1734,8 @@ void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt,
                                start_addr, end_addr);
        mmu_notifier_invalidate_range_start(&range);
        do {
-               unmap_single_vma(tlb, vma, start_addr, end_addr, &details);
+               unmap_single_vma(tlb, vma, start_addr, end_addr, &details,
+                                lock_vma);
        } while ((vma = mas_find(&mas, end_addr - 1)) != NULL);
        mmu_notifier_invalidate_range_end(&range);
 }
@@ -1763,7 +1764,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
        update_hiwater_rss(vma->vm_mm);
        mmu_notifier_invalidate_range_start(&range);
        do {
-               unmap_single_vma(&tlb, vma, start, range.end, NULL);
+               unmap_single_vma(&tlb, vma, start, range.end, NULL, false);
        } while ((vma = mas_find(&mas, end - 1)) != NULL);
        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);
@@ -1790,7 +1791,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
        tlb_gather_mmu(&tlb, vma->vm_mm);
        update_hiwater_rss(vma->vm_mm);
        mmu_notifier_invalidate_range_start(&range);
-       unmap_single_vma(&tlb, vma, address, range.end, details);
+       unmap_single_vma(&tlb, vma, address, range.end, details, false);
        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);
 }
@@ -2559,7 +2560,7 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
 
        err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
        if (err)
-               untrack_pfn(vma, pfn, PAGE_ALIGN(size));
+               untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
        return err;
 }
 EXPORT_SYMBOL(remap_pfn_range);

diff --git a/mm/memremap.c b/mm/memremap.c
index 08cbf54fe037081af32eed15c1f98dfdeb2ded8e..2f88f43d4a01793d5cbae343ca3c85bd035d5019 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -129,7 +129,7 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
        }
        mem_hotplug_done();
 
-       untrack_pfn(NULL, PHYS_PFN(range->start), range_len(range));
+       untrack_pfn(NULL, PHYS_PFN(range->start), range_len(range), true);
        pgmap_array_delete(range);
 }
 
@@ -276,7 +276,7 @@ err_add_memory:
        if (!is_private)
                kasan_remove_zero_shadow(__va(range->start), range_len(range));
 err_kasan:
-       untrack_pfn(NULL, PHYS_PFN(range->start), range_len(range));
+       untrack_pfn(NULL, PHYS_PFN(range->start), range_len(range), true);
 err_pfn_remap:
        pgmap_array_delete(range);
        return error;

diff --git a/mm/mmap.c b/mm/mmap.c
index f76da8580f877c1ef1f11d9e17602df271d34251..d374a16cbb9c7169f1b8a0c1cedcff819d0e76b1 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2219,7 +2219,7 @@ static void unmap_region(struct mm_struct *mm, struct maple_tree *mt,
        lru_add_drain();
        tlb_gather_mmu(&tlb, mm);
        update_hiwater_rss(mm);
-       unmap_vmas(&tlb, mt, vma, start, end);
+       unmap_vmas(&tlb, mt, vma, start, end, lock_vma);
        free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
                                 next ? next->vm_start : USER_PGTABLES_CEILING,
                                 lock_vma);
@@ -3141,7 +3141,7 @@ void exit_mmap(struct mm_struct *mm)
        tlb_gather_mmu_fullmm(&tlb, mm);
        /* update_hiwater_rss(mm) here? but nobody should be looking */
        /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
-       unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX);
+       unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX, false);
        mmap_read_unlock(mm);
 
        /*