mm/mmap: Change unmapped_area and unmapped_area_topdown to use maple

tree

Use the new maple tree data structure to find an unmapped area.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/mm/mmap.c b/mm/mmap.c
index 6a04b9c2f0553a3dff796d9e0a0e90622dff3151..726a6fac727f3427c2753a22a81b717fee25b324 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2058,23 +2058,21 @@ static inline unsigned long gap_end_offset(struct vm_unmapped_area_info *info,
        return (addr - info->align_offset) & info->align_mask;
 }
 
+/* unmapped_area() Find an area between the low_limit and the high_limit with
+ * the correct alignment and offset, all from @info. Note: current->mm is used
+ * for the search.
+ *
+ * @info: The unmapped area information including the range (low_limit -
+ * hight_limit), the alignment offset and mask.
+ *
+ * Return: A memory address or -ENOMEM.
+ */
 static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
 {
-       /*
-        * We implement the search by looking for an rbtree node that
-        * immediately follows a suitable gap. That is,
-        * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
-        * - gap_end   = vma->vm_start        >= info->low_limit  + length;
-        * - gap_end - gap_start >= length
-        */
-
-       struct mm_struct *mm = current->mm;
-       struct vm_area_struct *vma;
-       unsigned long length, low_limit, high_limit, gap_start, gap_end;
        unsigned long gap;
-       MA_STATE(mas, &mm->mm_mt, 0, 0);
+       MA_STATE(mas, &current->mm->mm_mt, 0, 0);
 
-       length = info->length;
+       validate_mm(current->mm);
 
        rcu_read_lock();
        do {
@@ -2085,116 +2083,26 @@ static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
                }
                gap = mas.index + gap_start_offset(info, mas.index);
        } while (gap > mas.last + 1 - info->length);
-
        rcu_read_unlock();
-       gap = mas.index;
-       gap += (info->align_offset - gap) & info->align_mask;
-
-       /* Adjust search limits by the desired length */
-       if (info->high_limit < length)
-               return -ENOMEM;
-       high_limit = info->high_limit - length;
-
-       if (info->low_limit > high_limit)
-               return -ENOMEM;
-       low_limit = info->low_limit + length;
-
-       /* Check if rbtree root looks promising */
-       if (RB_EMPTY_ROOT(&mm->mm_rb))
-               goto check_highest;
-       vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
-       if (vma->rb_subtree_gap < length)
-               goto check_highest;
-
-       while (true) {
-               /* Visit left subtree if it looks promising */
-               gap_end = vm_start_gap(vma);
-               if (gap_end >= low_limit && vma->vm_rb.rb_left) {
-                       struct vm_area_struct *left =
-                               rb_entry(vma->vm_rb.rb_left,
-                                        struct vm_area_struct, vm_rb);
-                       if (left->rb_subtree_gap >= length) {
-                               vma = left;
-                               continue;
-                       }
-               }
-
-               gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
-               gap_start += gap_start_offset(info, gap_start);
-check_current:
-               /* Check if current node has a suitable gap */
-               if (gap_start > high_limit)
-                       return -ENOMEM;
-               if (gap_end >= low_limit &&
-                   gap_end > gap_start && gap_end - gap_start >= length)
-                       goto found;
-
-               /* Visit right subtree if it looks promising */
-               if (vma->vm_rb.rb_right) {
-                       struct vm_area_struct *right =
-                               rb_entry(vma->vm_rb.rb_right,
-                                        struct vm_area_struct, vm_rb);
-                       if (right->rb_subtree_gap >= length) {
-                               vma = right;
-                               continue;
-                       }
-               }
-
-               /* Go back up the rbtree to find next candidate node */
-               while (true) {
-                       struct rb_node *prev = &vma->vm_rb;
-                       if (!rb_parent(prev))
-                               goto check_highest;
-                       vma = rb_entry(rb_parent(prev),
-                                      struct vm_area_struct, vm_rb);
-                       if (prev == vma->vm_rb.rb_left) {
-                               gap_start = vm_end_gap(vma->vm_prev);
-                               gap_start += gap_start_offset(info, gap_start);
-                               gap_end = vm_start_gap(vma);
-                               goto check_current;
-                       }
-               }
-       }
-
-check_highest:
-       /* Check highest gap, which does not precede any rbtree node */
-       gap_start = mm->highest_vm_end;
-       gap_start += gap_start_offset(info, gap_start);
-       gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
-       if (gap_start > high_limit)
-               return -ENOMEM;
-
-found:
-       /* We found a suitable gap. Clip it with the original low_limit. */
-       if (gap_start < info->low_limit) {
-               gap_start = info->low_limit;
-               gap_start += gap_start_offset(info, gap_start);
-       }
 
-       VM_BUG_ON(gap_start + info->length > info->high_limit);
-       VM_BUG_ON(gap_start + info->length > gap_end);
-
-       VM_BUG_ON(gap != gap_start);
-       return gap_start;
-}
-
-static inline unsigned long top_area_aligned(struct vm_unmapped_area_info *info,
-                                            unsigned long end)
-{
-       return (end - info->length - info->align_offset) & (~info->align_mask);
+       return gap;
 }
 
+/* unmapped_area_topdown() Find an area between the low_limit and the
+ * high_limit with * the correct alignment and offset at the highest available
+ * address, all from * @info. Note: current->mm is used for the search.
+ *
+ * @info: The unmapped area information including the range (low_limit -
+ * hight_limit), the alignment offset and mask.
+ *
+ * Return: A memory address or -ENOMEM.
+ */
 static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
 {
-       struct mm_struct *mm = current->mm;
-       struct vm_area_struct *vma = NULL;
-       unsigned long length, low_limit, high_limit, gap_start, gap_end;
        unsigned long gap;
+       MA_STATE(mas, &current->mm->mm_mt, 0, 0);
 
-       MA_STATE(mas, &mm->mm_mt, 0, 0);
-       validate_mm_mt(mm);
-
-       length = info->length;
+       validate_mm(current->mm);
 
        rcu_read_lock();
        do {
@@ -2207,121 +2115,8 @@ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
                        info->length + 1;
        } while (mas.index > gap);
        rcu_read_unlock();
-       /*
-        * Adjust search limits by the desired length.
-        * See implementation comment at top of unmapped_area().
-        */
-       gap_end = info->high_limit;
-       gap_end -= gap_end_offset(info, gap_end);
-       if (gap_end < length)
-               return -ENOMEM;
-       high_limit = gap_end - length;
-
-       if (info->low_limit > high_limit)
-               return -ENOMEM;
-       low_limit = info->low_limit + length;
-
-       /* Check highest gap, which does not precede any rbtree node */
-       gap_start = mm->highest_vm_end;
-       if (gap_start <= high_limit)
-               goto found_highest;
-
-       /* Check if rbtree root looks promising */
-       if (RB_EMPTY_ROOT(&mm->mm_rb))
-               return -ENOMEM;
-       vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
-       if (vma->rb_subtree_gap < length)
-               return -ENOMEM;
-
-       while (true) {
-               /* Visit right subtree if it looks promising */
-               gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
-               if (gap_start <= high_limit && vma->vm_rb.rb_right) {
-                       struct vm_area_struct *right =
-                               rb_entry(vma->vm_rb.rb_right,
-                                        struct vm_area_struct, vm_rb);
-                       if (right->rb_subtree_gap >= length) {
-                               vma = right;
-                               continue;
-                       }
-               }
-
-check_current:
-               /* Check if current node has a suitable gap */
-               gap_end = vm_start_gap(vma);
-               gap_end -= gap_end_offset(info, gap_end);
-               if (gap_end < low_limit)
-                       return -ENOMEM;
-               if (gap_start <= high_limit &&
-                   gap_end > gap_start && gap_end - gap_start >= length)
-                       goto found;
-
-               /* Visit left subtree if it looks promising */
-               if (vma->vm_rb.rb_left) {
-                       struct vm_area_struct *left =
-                               rb_entry(vma->vm_rb.rb_left,
-                                        struct vm_area_struct, vm_rb);
-                       if (left->rb_subtree_gap >= length) {
-                               vma = left;
-                               continue;
-                       }
-               }
 
-               /* Go back up the rbtree to find next candidate node */
-               while (true) {
-                       struct rb_node *prev = &vma->vm_rb;
-                       if (!rb_parent(prev))
-                               return -ENOMEM;
-                       vma = rb_entry(rb_parent(prev),
-                                      struct vm_area_struct, vm_rb);
-                       if (prev == vma->vm_rb.rb_right) {
-                               gap_start = vma->vm_prev ?
-                                       vm_end_gap(vma->vm_prev) : 0;
-                               goto check_current;
-                       }
-               }
-       }
-
-found:
-       /* We found a suitable gap. Clip it with the original high_limit. */
-       if (gap_end > info->high_limit) {
-               gap_end = info->high_limit;
-               gap_end -= gap_end_offset(info, gap_end);
-       }
-
-found_highest:
-       /* Compute highest gap address at the desired alignment */
-       gap_end -= info->length;
-
-       VM_BUG_ON(gap_end < info->low_limit);
-       VM_BUG_ON(gap_end < gap_start);
-
-       if (gap != gap_end) {
-               pr_err("%s: %px Gap was found: mt %lu gap_end %lu\n", __func__,
-                      mm, gap, gap_end);
-               pr_err("window was %lu - %lu size %lu\n", info->high_limit,
-                      info->low_limit, length);
-               pr_err("mas.min %lu max %lu mas.last %lu\n", mas.min, mas.max,
-                      mas.last);
-               pr_err("mas.index %lu align mask %lu offset %lu\n", mas.index,
-                      info->align_mask, info->align_offset);
-               pr_err("rb_find_vma find on %lu => %px (%px)\n", mas.index,
-                      find_vma(mm, mas.index), vma);
-#if defined(CONFIG_DEBUG_MAPLE_TREE)
-               mt_dump(&mm->mm_mt);
-#endif
-               {
-                       struct vm_area_struct *dv = mm->mmap;
-
-                       while (dv) {
-                               printk("vma %px %lu-%lu\n", dv, dv->vm_start, dv->vm_end);
-                               dv = dv->vm_next;
-                       }
-               }
-               VM_BUG_ON(gap != gap_end);
-       }
-
-       return gap_end;
+       return gap;
 }
 
 /*