static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
unsigned long newbrk, unsigned long oldbrk,
- struct list_head *uf, unsigned long max);
-static int do_brk_flags(struct ma_state *mas, struct vm_area_struct **brkvma,
+ struct list_head *uf);
+static int do_brk_flags(struct ma_state *mas, struct ma_state *ma_prev,
+ struct vm_area_struct **brkvma,
unsigned long addr, unsigned long request,
unsigned long flags);
SYSCALL_DEFINE1(brk, unsigned long, brk)
bool downgraded = false;
LIST_HEAD(uf);
MA_STATE(mas, &mm->mm_mt, 0, 0);
- struct ma_state ma_next;
+ struct ma_state ma_neighbour;
if (mmap_write_lock_killable(mm))
return -EINTR;
mas_set(&mas, newbrk);
brkvma = mas_walk(&mas);
- ma_next = mas;
- next = mas_next(&ma_next, -1);
if (brkvma) { // munmap necessary, there is something at newbrk.
/*
* Always allow shrinking brk.
* before calling do_brk_munmap().
*/
mm->brk = brk;
- mas.last = oldbrk - 1;
- ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf,
- next ? next->vm_start : USER_PGTABLES_CEILING);
+ ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf);
if (ret == 1) {
downgraded = true;
goto success;
mm->brk = origbrk;
goto out;
}
+ ma_neighbour = mas;
+ next = mas_next(&ma_neighbour, newbrk + PAGE_SIZE + stack_guard_gap);
/* Only check if the next VMA is within the stack_guard_gap of the
* expansion area */
/* Check against existing mmap mappings. */
if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
goto out;
- brkvma = mas_prev(&mas, mm->start_brk);
+ brkvma = mas_prev(&ma_neighbour, mm->start_brk);
if (brkvma && (brkvma->vm_start >= oldbrk))
goto out; // Trying to map over another vma.
/* Ok, looks good - let it rip. */
- if (do_brk_flags(&mas, &brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
+ if (do_brk_flags(&mas, &ma_neighbour, &brkvma, oldbrk,
+ newbrk - oldbrk, 0) < 0)
goto out;
mm->brk = brk;
{
int bug = 0;
int i = 0;
- unsigned long highest_address = 0;
struct vm_area_struct *vma;
MA_STATE(mas, &mm->mm_mt, 0, 0);
anon_vma_unlock_read(anon_vma);
}
#endif
- highest_address = vm_end_gap(vma);
i++;
}
if (i != mm->map_count) {
pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i);
bug = 1;
}
- if (highest_address != mm->highest_vm_end) {
- pr_emerg("mm->highest_vm_end %lx, found %lx\n",
- mm->highest_vm_end, highest_address);
- bug = 1;
- }
VM_BUG_ON_MM(bug, mm);
}
#else // !CONFIG_DEBUG_MAPLE_TREE
*
* Returns: True if there is an overlapping VMA, false otherwise
*/
-static bool range_has_overlap(struct mm_struct *mm, unsigned long start,
+static inline bool range_has_overlap(struct mm_struct *mm, unsigned long start,
unsigned long end, struct vm_area_struct **pprev)
{
struct vm_area_struct *existing;
}
static void vma_mas_link(struct mm_struct *mm, struct vm_area_struct *vma,
- struct ma_state *mas, struct vm_area_struct *prev)
+ struct ma_state *mas)
{
struct address_space *mapping = NULL;
}
vma_mas_store(vma, mas);
- __vma_link_list(mm, vma, prev);
__vma_link_file(vma);
if (mapping)
mm->map_count++;
validate_mm(mm);
}
-static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev)
+static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
{
struct address_space *mapping = NULL;
}
vma_mt_store(mm, vma);
- __vma_link_list(mm, vma, prev);
__vma_link_file(vma);
if (mapping)
* Helper for vma_adjust() in the split_vma insert case: insert a vma into the
* mm's list and the mm tree. It has already been inserted into the interval tree.
*/
-static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
+static inline void __insert_vm_struct(struct mm_struct *mm,
+ struct vm_area_struct *vma)
{
- struct vm_area_struct *prev;
-
- if (range_has_overlap(mm, vma->vm_start, vma->vm_end, &prev))
+ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
BUG();
vma_mt_store(mm, vma);
- __vma_link_list(mm, vma, prev);
mm->map_count++;
}
/* Expanding over the next vma */
if (remove_next) {
/* Remove from mm linked list - also updates highest_vm_end */
- __vma_unlink_list(mm, next);
-
if (file)
__remove_shared_vm_struct(next, file, mapping);
-
- } else if (!next) {
- mm->highest_vm_end = vm_end_gap(vma);
}
if (anon_vma) {
else
vma_changed = true;
vma->vm_end = end;
- if (!next)
- mm->highest_vm_end = vm_end_gap(vma);
}
if (vma_changed)
}
if (remove_next) {
- __vma_unlink_list(mm, next);
if (file)
__remove_shared_vm_struct(next, file, mapping);
} else if (insert) {
remove_next = 1;
end = next->vm_end;
goto again;
- } else if (!next) {
- /*
- * If remove_next == 2 we obviously can't
- * reach this path.
- *
- * If remove_next == 3 we can't reach this
- * path because pre-swap() next is always not
- * NULL. pre-swap() "next" is not being
- * removed and its next->vm_end is not altered
- * (and furthermore "end" already matches
- * next->vm_end in remove_next == 3).
- *
- * We reach this only in the remove_next == 1
- * case if the "next" vma that was removed was
- * the highest vma of the mm. However in such
- * case next->vm_end == "end" and the extended
- * "vma" has vma->vm_end == next->vm_end so
- * mm->highest_vm_end doesn't need any update
- * in remove_next == 1 case.
- */
- VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
}
}
if (insert && file)
return 0;
}
-/*
- * vma_next_wrap() - Get the next VMA of the first.
- * @mm: The mm_struct.
- * @vma: The current vma.
- *
- * If @vma is NULL, return the first vma in the mm.
- *
- * Returns: The next VMA after @vma.
- */
-static inline struct vm_area_struct *vma_next_wrap(struct mm_struct *mm,
- struct vm_area_struct *vma)
-{
- if (!vma)
- return mm->mmap;
-
- return vma_next(mm, vma);
-}
-
/*
* Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
* whether that can be merged with its predecessor or its successor.
if (vm_flags & VM_SPECIAL)
return NULL;
- next = vma_next_wrap(mm, prev);
+ if (!prev)
+ next = find_vma(mm, 0);
+ else
+ next = vma_next(mm, prev);
+
area = next;
if (area && area->vm_end == end) /* cases 6, 7, 8 */
next = vma_next(mm, next);
vma->vm_end = address;
vma_mt_store(mm, vma);
anon_vma_interval_tree_post_update_vma(vma);
- if (!vma_next(mm, vma))
- mm->highest_vm_end = vm_end_gap(vma);
spin_unlock(&mm->page_table_lock);
perf_event_mmap(vma);
return -EPERM;
/* Enforce stack_guard_gap */
- prev = vma->vm_prev;
+ prev = vma_prev(mm, vma);
/* Check that both stack segments have the same anon_vma? */
if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
vma_is_accessible(prev)) {
tlb_gather_mmu(&tlb, mm, start, end);
update_hiwater_rss(mm);
unmap_vmas(&tlb, vma, mas, start, end);
- free_pgtables(&tlb, &ma_pgtb,
+ free_pgtables(&tlb, &ma_pgtb, vma,
prev ? prev->vm_end : FIRST_USER_ADDRESS,
max);
tlb_finish_mmu(&tlb, start, end);
/* Drop removed area from the tree */
mas_store_gfp(src, NULL, GFP_KERNEL);
/* Set the upper limit */
- if (!tmp) {
- mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
+ if (!tmp)
return USER_PGTABLES_CEILING;
- }
return tmp->vm_start;
}
mas->index = start;
mas_reset(mas);
} else {
- prev = vma->vm_prev;
+ prev = vma_prev(mm, vma);
}
if (vma->vm_end >= end)
/* Point of no return */
max = detach_range(mm, mas, &dst, prev, &last);
-#if 1
- /* Detach vmas from the MM linked list */
- vma->vm_prev = NULL;
- if (prev)
- prev->vm_next = last->vm_next;
- else
- mm->mmap = last->vm_next;
-
- if (last->vm_next) {
- last->vm_next->vm_prev = prev;
- last->vm_next = NULL;
- }
-#endif
/*
* Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
* VM_GROWSUP VMA. Such VMAs can change their size under
mas_reset(&dst);
mas_set(&dst, start);
+ vma = mas_walk(&dst);
unmap_region(mm, vma, &dst, start, end, prev, max);
/* Fix up all other VM information */
unsigned long max = USER_PGTABLES_CEILING;
pgoff_t vm_pgoff;
int error;
+ struct ma_state ma_prev;
MA_STATE(mas, &mm->mm_mt, addr, end - 1);
/* Check against address space limit. */
}
+ ma_prev = mas;
if (vm_flags & VM_SPECIAL) {
- prev = mas_prev(&mas, 0);
+ prev = mas_prev(&ma_prev, 0);
goto cannot_expand;
}
/* Attempt to expand an old mapping */
/* Check next */
- next = mas_next(&mas, ULONG_MAX);
+ next = mas_next(&ma_prev, ULONG_MAX);
if (next) {
max = next->vm_start;
}
/* Check prev */
- prev = mas_prev(&mas, 0);
+ prev = mas_prev(&ma_prev, 0);
if (prev && prev->vm_end == addr && !vma_policy(prev) &&
can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
NULL_VM_UFFD_CTX)) {
/* Actually expand, if possible */
if (vma &&
- !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) {
+ !vma_expand(&ma_prev, vma, merge_start, merge_end, vm_pgoff, next)) {
khugepaged_enter_vma_merge(prev, vm_flags);
goto expanded;
}
pgoff, NULL_VM_UFFD_CTX))) {
merge_start = prev->vm_start;
vm_pgoff = prev->vm_pgoff;
- if (!vma_expand(&mas, prev, merge_start, merge_end,
+ if (!vma_expand(&ma_prev, prev, merge_start, merge_end,
vm_pgoff, next)) {
/* ->mmap() can change vma->vm_file and fput the original file. So
* fput the vma->vm_file here or we would add an extra fput for file
goto free_vma;
}
- /*
- * mas was called for the prev vma, and that may not be the correct
- * location for the vma being inserted, but is is before that location
- * and so the call to vma_mas_link()->vma_mas_store()->mas_store_gfp()
- * will detect the write as a spanning store and reset mas if necessary.
- */
- mas.index = mas.last = addr;
- mas_walk(&mas);
- vma_mas_link(mm, vma, &mas, prev);
+ vma_mas_link(mm, vma, &mas);
/* Once vma denies write, undo our temporary denial count */
if (file) {
unmap_writable:
fput(file);
mas.index = mas.last = addr;
- mas_walk(&mas);
+ vma = mas_walk(&mas);
/* Undo any partial mapping done by a device driver. */
unmap_region(mm, vma, &mas, vma->vm_start, vma->vm_end, prev, max);
charged = 0;
*/
static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
unsigned long newbrk, unsigned long oldbrk,
- struct list_head *uf, unsigned long max)
+ struct list_head *uf)
{
struct mm_struct *mm = vma->vm_mm;
- struct vm_area_struct unmap;
+ struct vm_area_struct unmap, *next;
unsigned long unmap_pages;
int ret;
+ struct ma_state ma_next;
arch_unmap(mm, newbrk, oldbrk);
vma_init(&unmap, mm);
unmap.vm_start = newbrk;
unmap.vm_end = oldbrk;
+ unmap.vm_pgoff = newbrk >> PAGE_SHIFT;
ret = userfaultfd_unmap_prep(&unmap, newbrk, oldbrk, uf);
if (ret)
return ret;
}
mmap_write_downgrade(mm);
- unmap_region(mm, &unmap, mas, newbrk, oldbrk, vma, max);
+ ma_next = *mas;
+ next = mas_next(&ma_next, -1);
+ unmap_region(mm, &unmap, mas, newbrk, oldbrk, vma,
+ next ? next->vm_start : 0);
/* Statistics */
vm_stat_account(mm, unmap.vm_flags, -unmap_pages);
if (unmap.vm_flags & VM_ACCOUNT)
* do not match then create a new anonymous VMA. Eventually we may be able to
* do some brk-specific accounting here.
*/
-static int do_brk_flags(struct ma_state *mas, struct vm_area_struct **brkvma,
+static int do_brk_flags(struct ma_state *mas, struct ma_state *ma_prev,
+ struct vm_area_struct **brkvma,
unsigned long addr, unsigned long len,
unsigned long flags)
{
struct mm_struct *mm = current->mm;
- struct vm_area_struct *prev = NULL, *vma;
+ struct vm_area_struct *vma;
int error;
unsigned long mapped_addr;
if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
return -ENOMEM;
- mas->last = addr + len - 1;
if (*brkvma) {
vma = *brkvma;
/* Expand the existing vma if possible; almost never a singular
if ((!vma->anon_vma ||
list_is_singular(&vma->anon_vma_chain)) &&
((vma->vm_flags & ~VM_SOFTDIRTY) == flags)){
- mas->index = vma->vm_start;
+ ma_prev->index = vma->vm_start;
+ ma_prev->last = addr + len - 1;
vma_adjust_trans_huge(vma, addr, addr + len, 0);
if (vma->anon_vma) {
}
vma->vm_end = addr + len;
vma->vm_flags |= VM_SOFTDIRTY;
- if (mas_store_gfp(mas, vma, GFP_KERNEL))
+ if (mas_store_gfp(ma_prev, vma, GFP_KERNEL))
goto mas_mod_fail;
if (vma->anon_vma) {
anon_vma_interval_tree_post_update_vma(vma);
khugepaged_enter_vma_merge(vma, flags);
goto out;
}
- prev = vma;
}
- mas->index = addr;
- mas_walk(mas);
+ mas->last = addr + len - 1;
/* create a vma struct for an anonymous mapping */
vma = vm_area_alloc(mm);
if (!vma)
if (vma_mas_store(vma, mas))
goto mas_store_fail;
- if (!prev)
- prev = mas_prev(mas, 0);
-
- __vma_link_list(mm, vma, prev);
mm->map_count++;
*brkvma = vma;
out:
// This vma left intentionally blank.
mas_walk(&mas);
- ret = do_brk_flags(&mas, &vma, addr, len, flags);
+ ret = do_brk_flags(&mas, &mas, &vma, addr, len, flags);
mmap_write_unlock(mm);
populate = ((mm->def_flags & VM_LOCKED) != 0);
if (populate && !ret)
struct mmu_gather tlb;
struct vm_area_struct *vma;
unsigned long nr_accounted = 0;
+ struct ma_state mas2;
MA_STATE(mas, &mm->mm_mt, FIRST_USER_ADDRESS, FIRST_USER_ADDRESS);
/* mm's last user has gone, and its about to be pulled down */
arch_exit_mmap(mm);
- vma = mm->mmap;
+ vma = mas_find(&mas, -1);
if (!vma) /* Can happen if dup_mmap() received an OOM */
return;
+ mas2 = mas;
+ mas_reset(&mas);
+ mas_set(&mas, FIRST_USER_ADDRESS);
+
lru_add_drain();
flush_cache_mm(mm);
tlb_gather_mmu(&tlb, mm, 0, -1);
/* update_hiwater_rss(mm) here? but nobody should be looking */
/* Use 0 here to ensure all VMAs in the mm are unmapped */
unmap_vmas(&tlb, vma, &mas, 0, -1);
- mas_reset(&mas);
- mas_set(&mas, FIRST_USER_ADDRESS);
- free_pgtables(&tlb, &mas, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
+ free_pgtables(&tlb, &mas2, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
tlb_finish_mmu(&tlb, 0, -1);
/*
*/
int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
{
- struct vm_area_struct *prev;
-
- if (range_has_overlap(mm, vma->vm_start, vma->vm_end, &prev))
+ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
return -ENOMEM;
if ((vma->vm_flags & VM_ACCOUNT) &&
vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
}
- vma_link(mm, vma, prev);
+ vma_link(mm, vma);
return 0;
}
get_file(new_vma->vm_file);
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
- vma_link(mm, new_vma, prev);
+ vma_link(mm, new_vma);
*need_rmap_locks = false;
}
return new_vma;
projects / users / jedix / linux-maple.git / commitdiff