mm/vma: Track start and end for munmap in vma_munmap_struct

Set the start and end address for munmap when the prev and next are
gathered.  This is needed to avoid incorrect addresses being used during
the vms_complete_munmap_vmas() function if the prev/next vma are
expanded.

Add a new helper vms_complete_pte_clear(), which is needed later and
will avoid growing the argument list to unmap_region() beyond the 9 it
already has.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>

diff --git a/mm/vma.c b/mm/vma.c
index 3bfc4934999628f169dbf86c00112e562faa2703..efacc8741e3ee04fc9c0f13605d3e5fdd6d29358 100644 (file)
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -646,6 +646,26 @@ again:
                uprobe_mmap(vp->insert);
 }
 
+static void vms_complete_pte_clear(struct vma_munmap_struct *vms,
+               struct ma_state *mas_detach, bool mm_wr_locked)
+{
+       struct mmu_gather tlb;
+
+       /*
+        * We can free page tables without write-locking mmap_lock because VMAs
+        * were isolated before we downgraded mmap_lock.
+        */
+       mas_set(mas_detach, 1);
+       lru_add_drain();
+       tlb_gather_mmu(&tlb, vms->mm);
+       update_hiwater_rss(vms->mm);
+       unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end, vms->vma_count, mm_wr_locked);
+       mas_set(mas_detach, 1);
+       /* start and end may be different if there is no prev or next vma. */
+       free_pgtables(&tlb, mas_detach, vms->vma, vms->unmap_start, vms->unmap_end, mm_wr_locked);
+       tlb_finish_mmu(&tlb);
+}
+
 /*
  * vms_complete_munmap_vmas() - Finish the munmap() operation
  * @vms: The vma munmap struct
@@ -667,13 +687,7 @@ void vms_complete_munmap_vmas(struct vma_munmap_struct *vms,
        if (vms->unlock)
                mmap_write_downgrade(mm);
 
-       /*
-        * We can free page tables without write-locking mmap_lock because VMAs
-        * were isolated before we downgraded mmap_lock.
-        */
-       mas_set(mas_detach, 1);
-       unmap_region(mm, mas_detach, vms->vma, vms->prev, vms->next,
-                    vms->start, vms->end, vms->vma_count, !vms->unlock);
+       vms_complete_pte_clear(vms, mas_detach, !vms->unlock);
        /* Update high watermark before we lower total_vm */
        update_hiwater_vm(mm);
        /* Stat accounting */
@@ -746,6 +760,8 @@ int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
                        goto start_split_failed;
        }
        vms->prev = vma_prev(vms->vmi);
+       if (vms->prev)
+               vms->unmap_start = vms->prev->vm_end;
 
        /*
         * Detach a range of VMAs from the mm. Using next as a temp variable as
@@ -810,6 +826,8 @@ int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
        }
 
        vms->next = vma_next(vms->vmi);
+       if (vms->next)
+               vms->unmap_end = vms->next->vm_start;
 
 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
        /* Make sure no VMAs are about to be lost. */

diff --git a/mm/vma.h b/mm/vma.h
index c85fc7c888a8e066df7cff537add43a93ce95561..7bc0f9e7751b2db27154906c9a40ccbc689a5570 100644 (file)
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -38,6 +38,8 @@ struct vma_munmap_struct {
        struct list_head *uf;           /* Userfaultfd list_head */
        unsigned long start;            /* Aligned start addr (inclusive) */
        unsigned long end;              /* Aligned end addr (exclusive) */
+       unsigned long unmap_start;      /* Unmap PTE start */
+       unsigned long unmap_end;        /* Unmap PTE end */
        int vma_count;                  /* Number of vmas that will be removed */
        unsigned long nr_pages;         /* Number of pages being removed */
        unsigned long locked_vm;        /* Number of locked pages */
@@ -108,6 +110,8 @@ static inline void init_vma_munmap(struct vma_munmap_struct *vms,
        vms->vma_count = 0;
        vms->nr_pages = vms->locked_vm = vms->nr_accounted = 0;
        vms->exec_vm = vms->stack_vm = vms->data_vm = 0;
+       vms->unmap_start = FIRST_USER_ADDRESS;
+       vms->unmap_end = USER_PGTABLES_CEILING;
 }
 
 int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,