From 1815ae26adfbefd2d0f5d4b7e932e6faee48d8ba Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Fri, 5 Aug 2022 16:07:25 -0400 Subject: [PATCH] mm/mmap: Refactor locking out of __vma_adjust() Move the locking into lock_vma() and unlock_vma() for use elsewhere Signed-off-by: Liam R. Howlett --- mm/internal.h | 13 +++ mm/mmap.c | 234 +++++++++++++++++++++++++++++--------------------- 2 files changed, 150 insertions(+), 97 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 15e8cb118832..e8dc14625b2b 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -860,4 +860,17 @@ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); +/* + * VMA lock generalization + */ +struct vma_locking { + struct vm_area_struct *vma; + struct vm_area_struct *adj_next; + struct file *file; + struct address_space *mapping; + struct anon_vma *anon_vma; + struct vm_area_struct *insert; + struct vm_area_struct *remove; + struct vm_area_struct *remove2; +}; #endif /* __MM_INTERNAL_H */ diff --git a/mm/mmap.c b/mm/mmap.c index 3191d0da0713..13c0390565aa 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -606,6 +606,126 @@ nomem: return -ENOMEM; } +/* + * lock_vma() - Helper function for locking VMAs prior to altering + * @vl: The initialized vma_locking struct + */ +static inline void lock_vma(struct vma_locking *vl) +{ + if (vl->file) { + uprobe_munmap(vl->vma, vl->vma->vm_start, vl->vma->vm_end); + + if (vl->adj_next) + uprobe_munmap(vl->adj_next, vl->adj_next->vm_start, + vl->adj_next->vm_end); + + i_mmap_lock_write(vl->mapping); + if (vl->insert && vl->insert->vm_file) { + /* + * Put into interval tree now, so instantiated pages + * are visible to arm/parisc __flush_dcache_page + * throughout; but we cannot insert into address + * space until vma start or end is updated. + */ + __vma_link_file(vl->insert, + vl->insert->vm_file->f_mapping); + } + } + + if (vl->anon_vma) { + anon_vma_lock_write(vl->anon_vma); + anon_vma_interval_tree_pre_update_vma(vl->vma); + if (vl->adj_next) + anon_vma_interval_tree_pre_update_vma(vl->adj_next); + } + + if (vl->file) { + flush_dcache_mmap_lock(vl->mapping); + vma_interval_tree_remove(vl->vma, &vl->mapping->i_mmap); + if (vl->adj_next) + vma_interval_tree_remove(vl->adj_next, + &vl->mapping->i_mmap); + } + +} + +/* + * unlock_vma - Helper function for unlocking after altering VMAs + * @vl: The vma_locking struct + * @mas: The maple state struct + * @mm: The mm_struct + */ +static inline void unlock_vma(struct vma_locking *vl, struct ma_state *mas, + struct mm_struct *mm) +{ + if (vl->file) { + if (vl->adj_next) + vma_interval_tree_insert(vl->adj_next, + &vl->mapping->i_mmap); + vma_interval_tree_insert(vl->vma, &vl->mapping->i_mmap); + flush_dcache_mmap_unlock(vl->mapping); + } + + if (vl->remove && vl->file) { + __remove_shared_vm_struct(vl->remove, vl->file, vl->mapping); + if (vl->remove2) + __remove_shared_vm_struct(vl->remove2, vl->file, + vl->mapping); + } else if (vl->insert) { + /* + * split_vma has split insert from vma, and needs + * us to insert it before dropping the locks + * (it may either follow vma or precede it). + */ + mas_reset(mas); + vma_mas_store(vl->insert, mas); + mm->map_count++; + } + + if (vl->anon_vma) { + anon_vma_interval_tree_post_update_vma(vl->vma); + if (vl->adj_next) + anon_vma_interval_tree_post_update_vma(vl->adj_next); + anon_vma_unlock_write(vl->anon_vma); + } + + if (vl->file) { + i_mmap_unlock_write(vl->mapping); + uprobe_mmap(vl->vma); + + if (vl->adj_next) + uprobe_mmap(vl->adj_next); + } + + if (vl->remove) { +again: + if (vl->file) { + uprobe_munmap(vl->remove, vl->remove->vm_start, + vl->remove->vm_end); + fput(vl->file); + } + if (vl->remove->anon_vma) + anon_vma_merge(vl->vma, vl->remove); + mm->map_count--; + mpol_put(vma_policy(vl->remove)); + if (!vl->remove2) + BUG_ON(vl->vma->vm_end < vl->remove->vm_end); + vm_area_free(vl->remove); + + /* + * In mprotect's case 6 (see comments on vma_merge), + * we must remove next_next too. + */ + if (vl->remove2) { + vl->remove = vl->remove2; + vl->remove2 = NULL; + goto again; + } + } + if (vl->insert && vl->file) + uprobe_mmap(vl->insert); +} + /* * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that * is already present in an i_mmap tree without adjusting the tree. @@ -618,10 +738,9 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, struct vm_area_struct *expand) { struct mm_struct *mm = vma->vm_mm; - struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); + struct vm_area_struct *next_next = NULL; + struct vm_area_struct *next = find_vma(mm, vma->vm_end); struct vm_area_struct *orig_vma = vma; - struct address_space *mapping = NULL; - struct rb_root_cached *root = NULL; struct anon_vma *anon_vma = NULL; struct file *file = vma->vm_file; bool vma_changed = false; @@ -629,6 +748,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, int remove_next = 0; MA_STATE(mas, &mm->mm_mt, 0, 0); struct vm_area_struct *exporter = NULL, *importer = NULL; + struct vma_locking vma_lock; if (next && !insert) { if (end >= next->vm_end) { @@ -724,39 +844,22 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, anon_vma != next->anon_vma); vma_adjust_trans_huge(orig_vma, start, end, adjust_next); - if (file) { - mapping = file->f_mapping; - root = &mapping->i_mmap; - uprobe_munmap(vma, vma->vm_start, vma->vm_end); - if (adjust_next) - uprobe_munmap(next, next->vm_start, next->vm_end); - - i_mmap_lock_write(mapping); - if (insert && insert->vm_file) { - /* - * Put into interval tree now, so instantiated pages - * are visible to arm/parisc __flush_dcache_page - * throughout; but we cannot insert into address - * space until vma start or end is updated. - */ - __vma_link_file(insert, insert->vm_file->f_mapping); - } - } - - if (anon_vma) { - anon_vma_lock_write(anon_vma); - anon_vma_interval_tree_pre_update_vma(vma); - if (adjust_next) - anon_vma_interval_tree_pre_update_vma(next); + memset(&vma_lock, 0, sizeof(vma_lock)); + vma_lock.vma = vma; + vma_lock.anon_vma = anon_vma; + vma_lock.file = file; + if (adjust_next) + vma_lock.adj_next = next; + if (file) + vma_lock.mapping = file->f_mapping; + vma_lock.insert = insert; + if (remove_next) { + vma_lock.remove = next; + vma_lock.remove2 = next_next; } - if (file) { - flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, root); - if (adjust_next) - vma_interval_tree_remove(next, root); - } + lock_vma(&vma_lock); if (start != vma->vm_start) { if ((vma->vm_start < start) && @@ -792,70 +895,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, vma_mas_store(next, &mas); } - if (file) { - if (adjust_next) - vma_interval_tree_insert(next, root); - vma_interval_tree_insert(vma, root); - flush_dcache_mmap_unlock(mapping); - } - - if (remove_next && file) { - __remove_shared_vm_struct(next, file, mapping); - if (remove_next == 2) - __remove_shared_vm_struct(next_next, file, mapping); - } else if (insert) { - /* - * split_vma has split insert from vma, and needs - * us to insert it before dropping the locks - * (it may either follow vma or precede it). - */ - mas_reset(&mas); - vma_mas_store(insert, &mas); - mm->map_count++; - } - - if (anon_vma) { - anon_vma_interval_tree_post_update_vma(vma); - if (adjust_next) - anon_vma_interval_tree_post_update_vma(next); - anon_vma_unlock_write(anon_vma); - } - - if (file) { - i_mmap_unlock_write(mapping); - uprobe_mmap(vma); - - if (adjust_next) - uprobe_mmap(next); - } - - if (remove_next) { -again: - if (file) { - uprobe_munmap(next, next->vm_start, next->vm_end); - fput(file); - } - if (next->anon_vma) - anon_vma_merge(vma, next); - mm->map_count--; - mpol_put(vma_policy(next)); - if (remove_next != 2) - BUG_ON(vma->vm_end < next->vm_end); - vm_area_free(next); - - /* - * In mprotect's case 6 (see comments on vma_merge), - * we must remove next_next too. - */ - if (remove_next == 2) { - remove_next = 1; - next = next_next; - goto again; - } - } - if (insert && file) - uprobe_mmap(insert); - + unlock_vma(&vma_lock, &mas, mm); mas_destroy(&mas); validate_mm(mm); -- 2.50.1