mm/vma: Change munmap to use vma_munmap_struct() for accounting and surrounding

vmas

Clean up the code by changing the munmap operation to use a structure
for the accounting and munmap variables.

Since remove_mt() is only called in one location and the contents will
be reduced to almost nothing.  The remains of the function can be added
to vms_complete_munmap_vmas().

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>

diff --git a/mm/vma.c b/mm/vma.c
index e1aee43a3dc4e2c4db347526ce76e95bbc49c4c8..58604fe3bd03c996f0be987982db385766256856 100644 (file)
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -103,7 +103,8 @@ static inline void init_vma_munmap(struct vma_munmap_struct *vms,
        vms->unlock = unlock;
        vms->uf = uf;
        vms->vma_count = 0;
-       vms->nr_pages = vms->locked_vm = 0;
+       vms->nr_pages = vms->locked_vm = vms->nr_accounted = 0;
+       vms->exec_vm = vms->stack_vm = vms->data_vm = 0;
 }
 
 /*
@@ -299,30 +300,6 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
        return __split_vma(vmi, vma, addr, new_below);
 }
 
-/*
- * Ok - we have the memory areas we should free on a maple tree so release them,
- * and do the vma updates.
- *
- * Called with the mm semaphore held.
- */
-static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
-{
-       unsigned long nr_accounted = 0;
-       struct vm_area_struct *vma;
-
-       /* Update high watermark before we lower total_vm */
-       update_hiwater_vm(mm);
-       mas_for_each(mas, vma, ULONG_MAX) {
-               long nrpages = vma_pages(vma);
-
-               if (vma->vm_flags & VM_ACCOUNT)
-                       nr_accounted += nrpages;
-               vm_stat_account(mm, vma->vm_flags, -nrpages);
-               remove_vma(vma, false);
-       }
-       vm_unacct_memory(nr_accounted);
-}
-
 /*
  * init_vma_prep() - Initializer wrapper for vma_prepare struct
  * @vp: The vma_prepare struct
@@ -722,7 +699,7 @@ static inline void abort_munmap_vmas(struct ma_state *mas_detach)
 static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms,
                struct ma_state *mas_detach)
 {
-       struct vm_area_struct *prev, *next;
+       struct vm_area_struct *vma;
        struct mm_struct *mm;
 
        mm = vms->mm;
@@ -731,21 +708,31 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms,
        if (vms->unlock)
                mmap_write_downgrade(mm);
 
-       prev = vma_iter_prev_range(vms->vmi);
-       next = vma_next(vms->vmi);
-       if (next)
-               vma_iter_prev_range(vms->vmi);
-
        /*
         * We can free page tables without write-locking mmap_lock because VMAs
         * were isolated before we downgraded mmap_lock.
         */
        mas_set(mas_detach, 1);
-       unmap_region(mm, mas_detach, vms->vma, prev, next, vms->start, vms->end,
-                    vms->vma_count, !vms->unlock);
-       /* Statistics and freeing VMAs */
+       unmap_region(mm, mas_detach, vms->vma, vms->prev, vms->next,
+                    vms->start, vms->end, vms->vma_count, !vms->unlock);
+       /* Update high watermark before we lower total_vm */
+       update_hiwater_vm(mm);
+       /* Stat accounting */
+       WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm) - vms->nr_pages);
+       mm->exec_vm -= vms->exec_vm;
+       mm->stack_vm -= vms->stack_vm;
+       mm->data_vm -= vms->data_vm;
+       /* Paranoid bookkeeping */
+       VM_WARN_ON(vms->exec_vm > mm->exec_vm);
+       VM_WARN_ON(vms->stack_vm > mm->stack_vm);
+       VM_WARN_ON(vms->data_vm > mm->data_vm);
+
+       /* Remove and clean up vmas */
        mas_set(mas_detach, 0);
-       remove_mt(mm, mas_detach);
+       mas_for_each(mas_detach, vma, ULONG_MAX)
+               remove_vma(vma, false);
+
+       vm_unacct_memory(vms->nr_accounted);
        validate_mm(mm);
        if (vms->unlock)
                mmap_read_unlock(mm);
@@ -799,18 +786,19 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
                if (error)
                        goto start_split_failed;
        }
+       vms->prev = vma_prev(vms->vmi);
 
        /*
         * Detach a range of VMAs from the mm. Using next as a temp variable as
         * it is always overwritten.
         */
-       next = vms->vma;
-       do {
+       for_each_vma_range(*(vms->vmi), next, vms->end) {
+               long nrpages;
+
                if (!can_modify_vma(next)) {
                        error = -EPERM;
                        goto modify_vma_failed;
                }
-
                /* Does it split the end? */
                if (next->vm_end > vms->end) {
                        error = __split_vma(vms->vmi, next, vms->end, 0);
@@ -824,8 +812,21 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
                        goto munmap_gather_failed;
 
                vma_mark_detached(next, true);
+               nrpages = vma_pages(next);
+
+               vms->nr_pages += nrpages;
                if (next->vm_flags & VM_LOCKED)
-                       vms->locked_vm += vma_pages(next);
+                       vms->locked_vm += nrpages;
+
+               if (next->vm_flags & VM_ACCOUNT)
+                       vms->nr_accounted += nrpages;
+
+               if (is_exec_mapping(next->vm_flags))
+                       vms->exec_vm += nrpages;
+               else if (is_stack_mapping(next->vm_flags))
+                       vms->stack_vm += nrpages;
+               else if (is_data_mapping(next->vm_flags))
+                       vms->data_vm += nrpages;
 
                if (unlikely(vms->uf)) {
                        /*
@@ -847,7 +848,9 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
                BUG_ON(next->vm_start < vms->start);
                BUG_ON(next->vm_start > vms->end);
 #endif
-       } for_each_vma_range(*(vms->vmi), next, vms->end);
+       }
+
+       vms->next = vma_next(vms->vmi);
 
 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
        /* Make sure no VMAs are about to be lost. */

diff --git a/mm/vma.h b/mm/vma.h
index cb67acf59012fb4670e2420929464206f1ded10a..cbf55e0e0c4fca741ba3a2db0ed6c0aef1b38dc4 100644 (file)
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -33,12 +33,18 @@ struct vma_munmap_struct {
        struct vma_iterator *vmi;
        struct mm_struct *mm;
        struct vm_area_struct *vma;     /* The first vma to munmap */
+       struct vm_area_struct *prev;    /* vma before the munmap area */
+       struct vm_area_struct *next;    /* vma after the munmap area */
        struct list_head *uf;           /* Userfaultfd list_head */
        unsigned long start;            /* Aligned start addr (inclusive) */
        unsigned long end;              /* Aligned end addr (exclusive) */
        int vma_count;                  /* Number of vmas that will be removed */
        unsigned long nr_pages;         /* Number of pages being removed */
        unsigned long locked_vm;        /* Number of locked pages */
+       unsigned long nr_accounted;     /* Number of VM_ACCOUNT pages */
+       unsigned long exec_vm;
+       unsigned long stack_vm;
+       unsigned long data_vm;
        bool unlock;                    /* Unlock after the munmap */
 };