sched: use maple tree iterator to walk VMAs

The linked list is slower than walking the VMAs using the maple tree.  We
can't use the VMA iterator here because it doesn't support moving to an
earlier position.

Link: https://lkml.kernel.org/r/20220404143501.2016403-50-Liam.Howlett@oracle.com
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d4bd299d67abfe84e56cf99693adc9a12a0ae27d..b8b7962b5832dad3e9ab14d1aacf75542d5be77b 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2722,6 +2722,7 @@ static void task_numa_work(struct callback_head *work)
        struct task_struct *p = current;
        struct mm_struct *mm = p->mm;
        u64 runtime = p->se.sum_exec_runtime;
+       MA_STATE(mas, &mm->mm_mt, 0, 0);
        struct vm_area_struct *vma;
        unsigned long start, end;
        unsigned long nr_pte_updates = 0;
@@ -2778,13 +2779,16 @@ static void task_numa_work(struct callback_head *work)
 
        if (!mmap_read_trylock(mm))
                return;
-       vma = find_vma(mm, start);
+       mas_set(&mas, start);
+       vma = mas_find(&mas, ULONG_MAX);
        if (!vma) {
                reset_ptenuma_scan(p);
                start = 0;
-               vma = mm->mmap;
+               mas_set(&mas, start);
+               vma = mas_find(&mas, ULONG_MAX);
        }
-       for (; vma; vma = vma->vm_next) {
+
+       for (; vma; vma = mas_find(&mas, ULONG_MAX)) {
                if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
                        is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
                        continue;