sched: use maple tree iterator to walk VMAs
authorMatthew Wilcox (Oracle) <willy@infradead.org>
Wed, 20 Jul 2022 02:17:59 +0000 (02:17 +0000)
committerLiam R. Howlett <Liam.Howlett@oracle.com>
Tue, 6 Sep 2022 19:44:25 +0000 (15:44 -0400)
The linked list is slower than walking the VMAs using the maple tree.  We
can't use the VMA iterator here because it doesn't support moving to an
earlier position.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
kernel/sched/fair.c

index d642e9ff28292072157b7f9d99a9b6168389806e..c0c0e691d42656e939985957a1a86b020ac70918 100644 (file)
@@ -2929,6 +2929,7 @@ static void task_numa_work(struct callback_head *work)
        struct task_struct *p = current;
        struct mm_struct *mm = p->mm;
        u64 runtime = p->se.sum_exec_runtime;
+       MA_STATE(mas, &mm->mm_mt, 0, 0);
        struct vm_area_struct *vma;
        unsigned long start, end;
        unsigned long nr_pte_updates = 0;
@@ -2985,13 +2986,16 @@ static void task_numa_work(struct callback_head *work)
 
        if (!mmap_read_trylock(mm))
                return;
-       vma = find_vma(mm, start);
+       mas_set(&mas, start);
+       vma = mas_find(&mas, ULONG_MAX);
        if (!vma) {
                reset_ptenuma_scan(p);
                start = 0;
-               vma = mm->mmap;
+               mas_set(&mas, start);
+               vma = mas_find(&mas, ULONG_MAX);
        }
-       for (; vma; vma = vma->vm_next) {
+
+       for (; vma; vma = mas_find(&mas, ULONG_MAX)) {
                if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
                        is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
                        continue;