{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ rcu_read_lock();
+ mas_for_each(&mas, vma, ULONG_MAX) {
unsigned long size = vma->vm_end - vma->vm_start;
if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
zap_page_range(vma, vma->vm_start, size);
#endif
}
+ rcu_read_unlock();
mmap_read_unlock(mm);
return 0;
{
struct vm_area_struct *vma;
unsigned long usize = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ mas_for_each(&mas, vma, ULONG_MAX)
usize += vma->vm_end - vma->vm_start;
return usize;
}
{
struct vm_area_struct *vma;
pgd_t *pgd;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* Flushing the whole cache on each cpu takes forever on
rp3440, etc. So, avoid it if the mm isn't too big. */
}
if (mm->context == mfsp(3)) {
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
flush_user_dcache_range_asm(vma->vm_start, vma->vm_end);
if (vma->vm_flags & VM_EXEC)
flush_user_icache_range_asm(vma->vm_start, vma->vm_end);
}
pgd = mm->pgd;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
unsigned long addr;
for (addr = vma->vm_start; addr < vma->vm_end;
void flush_tlb_mm(struct mm_struct *mm)
{
struct vm_area_struct *mp;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
if (!Hash) {
_tlbia();
* unmap_region or exit_mmap, but not from vmtruncate on SMP -
* but it seems dup_mmap is the only SMP case which gets here.
*/
- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next)
+ mas_for_each(&mas, mp, ULONG_MAX)
flush_range(mp->vm_mm, mp->vm_start, mp->vm_end);
}
EXPORT_SYMBOL(flush_tlb_mm);
unsigned long len)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
/*
* We don't try too hard, we just mark all the vma in that range
* VM_NOHUGEPAGE and split them.
*/
- vma = find_vma(mm, addr);
- /*
- * If the range is in unmapped range, just return
- */
- if (vma && ((addr + len) <= vma->vm_start))
- return;
-
- while (vma) {
- if (vma->vm_start >= (addr + len))
- break;
+ mas_for_each(&mas, vma, addr + len) {
vma->vm_flags |= VM_NOHUGEPAGE;
walk_page_vma(vma, &subpage_walk_ops, NULL);
- vma = vma->vm_next;
}
}
#else
struct vm_area_struct *vma;
struct file *exe_file;
struct mm_struct *mm = spu->mm;
+ MA_STATE(mas, &mm->mm_mt, spu_ref, spu_ref);
if (!mm)
goto out;
}
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
- if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
- continue;
- my_offset = spu_ref - vma->vm_start;
- if (!vma->vm_file)
- goto fail_no_image_cookie;
-
- pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n",
- my_offset, spu_ref, vma->vm_file);
- *offsetp = my_offset;
- break;
- }
+ vma = mas_walk(&mas);
+ if (!vma)
+ goto fail_no_image_cookie;
+
+ my_offset = spu_ref - vma->vm_start;
+ if (!vma->vm_file)
+ goto fail_no_image_cookie;
+ pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n",
+ my_offset, spu_ref, vma->vm_file);
+ *offsetp = my_offset;
*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
pr_debug("got dcookie for %pD\n", vma->vm_file);
static inline void thp_split_mm(struct mm_struct *mm)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
vma->vm_flags &= ~VM_HUGEPAGE;
vma->vm_flags |= VM_NOHUGEPAGE;
walk_page_vma(vma, &thp_split_walk_ops, NULL);
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int ret;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
MADV_UNMERGEABLE, &vma->vm_flags);
if (ret)
void flush_tlb_mm(struct mm_struct *mm)
{
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- while (vma != NULL) {
+ mas_for_each(&mas, vma, ULONG_MAX)
fix_range(mm, vma->vm_start, vma->vm_end, 0);
- vma = vma->vm_next;
- }
}
void force_flush_all(void)
{
struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- while (vma != NULL) {
+ mas_for_each(&mas, vma, ULONG_MAX)
fix_range(mm, vma->vm_start, vma->vm_end, 1);
- vma = vma->vm_next;
- }
}
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
unsigned long size = vma->vm_end - vma->vm_start;
if (vma_is_special_mapping(vma, &vvar_mapping))
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_write_lock(mm);
/*
* We could search vma near context.vdso, but it's a slowpath,
* so let's explicitly check all VMAs to be completely sure.
*/
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma_is_special_mapping(vma, &vdso_mapping) ||
vma_is_special_mapping(vma, &vvar_mapping)) {
mmap_write_unlock(mm);
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct vm_area_struct *vmm;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
if (flags & MAP_FIXED) {
/* We do not accept a shared mapping if it would violate
else
addr = PAGE_ALIGN(addr);
- for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
+ mas_for_each(&mas, vmm, ULONG_MAX) {
/* At this point: (!vmm || addr < vmm->vm_end). */
if (TASK_SIZE - len < addr)
return -ENOMEM;
struct vm_area_struct *vma;
int rc;
struct mm_struct *mm;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mm = get_mem_context(ctx);
if (mm == NULL) {
}
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
for (ea = vma->vm_start; ea < vma->vm_end;
ea = next_segment(ea, slb.vsid)) {
rc = copro_calculate_slb(mm, ea, &slb);
struct vm_area_struct *vma;
mmap_read_lock(mm);
- for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
-
- if (addr < vma->vm_start || addr >= vma->vm_end)
- continue;
-
+ vma = find_vma_intersection(mm, addr, addr + 1);
+ if (vma) {
if (vma->vm_file) {
cookie = fast_get_dcookie(&vma->vm_file->f_path);
*offset = (vma->vm_pgoff << PAGE_SHIFT) + addr -
/* must be an anonymous map */
*offset = addr;
}
-
- break;
- }
-
- if (!vma)
+ } else
cookie = INVALID_COOKIE;
+
mmap_read_unlock(mm);
return cookie;
static int __check_mem_type(struct vm_area_struct *vma, unsigned long end)
{
- while (vma && is_normal_memory(vma->vm_page_prot)) {
- if (vma->vm_end >= end)
- return 0;
- vma = vma->vm_next;
+ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_start, vma->vm_start);
+
+
+ mas_for_each(&mas, vma, end) {
+ if (!is_normal_memory(vma->vm_page_prot))
+ break;
}
+ if (!vma)
+ return 0;
+
return -EINVAL;
}
user_long_t *data;
user_long_t *start_end_ofs;
char *name_base, *name_curpos;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* *Estimated* file count and total data size needed */
count = mm->map_count;
name_base = name_curpos = ((char *)data) + names_ofs;
remaining = size - names_ofs;
count = 0;
- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
struct file *file;
const char *filename;
{
struct vm_area_struct *ret;
- ret = this_vma->vm_next;
+ ret = vma_next(this_vma->vm_mm, this_vma);
if (ret)
return ret;
if (this_vma == gate_vma)
* when the old and new regions overlap clear from new_end.
*/
free_pgd_range(&tlb, new_end, old_end, new_end,
- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING);
+ vma_next(mm, vma) ? vma_next(mm, vma)->vm_start : USER_PGTABLES_CEILING);
} else {
/*
* otherwise, clean from old_start; this is done to not touch
* for the others its just a little faster.
*/
free_pgd_range(&tlb, old_start, old_end, new_end,
- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING);
+ vma_next(mm, vma) ? vma_next(mm, vma)->vm_start : USER_PGTABLES_CEILING);
}
tlb_finish_mmu(&tlb, old_start, old_end);
GENRADIX(struct map_files_info) fa;
struct map_files_info *p;
int ret;
+ MA_STATE(mas, NULL, 0, 0);
genradix_init(&fa);
}
nr_files = 0;
+ mas.tree = &mm->mm_mt;
/*
* We need two passes here:
* routine might require mmap_lock taken in might_fault().
*/
- for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) {
+ pos = 2;
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!vma->vm_file)
continue;
if (++pos <= ctx->pos)
static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
{
struct proc_maps_private *priv = m->private;
- struct vm_area_struct *next, *vma = v;
+ struct vm_area_struct *next = NULL, *vma = v;
- if (vma == priv->tail_vma)
- next = NULL;
- else if (vma->vm_next)
- next = vma->vm_next;
- else
- next = priv->tail_vma;
+ if (vma != priv->tail_vma) {
+ next = vma_next(vma->vm_mm, vma);
+ if (!next)
+ next = priv->tail_vma;
+ }
*ppos = next ? next->vm_start : -1UL;
smap_gather_stats(vma, &mss, last_vma_end);
}
/* Case 2 above */
- vma = vma->vm_next;
+ vma = vma_next(mm, vma);
}
show_vma_header_prefix(m, priv->mm->mmap->vm_start,
return -ESRCH;
mm = get_task_mm(task);
if (mm) {
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
struct mmu_notifier_range range;
struct clear_refs_private cp = {
.type = type,
}
tlb_gather_mmu(&tlb, mm, 0, -1);
if (type == CLEAR_REFS_SOFT_DIRTY) {
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!(vma->vm_flags & VM_SOFTDIRTY))
continue;
mmap_read_unlock(mm);
count = -EINTR;
goto out_mm;
}
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_reset(&mas);
+ mas_set(&mas, 0);
+ mas_for_each(&mas, vma, ULONG_MAX) {
vma->vm_flags &= ~VM_SOFTDIRTY;
vma_set_page_prot(vma);
}
if (release_new_ctx) {
struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
}
+ }
mmap_write_unlock(mm);
userfaultfd_ctx_put(release_new_ctx);
unsigned long start, unsigned long end,
struct list_head *unmaps)
{
- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
+
+ mas_for_each(&mas, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
/* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
WRITE_ONCE(ctx->released, true);
*/
mmap_write_lock(mm);
prev = NULL;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
cond_resched();
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
bool found;
bool basic_ioctls;
unsigned long start, end, vma_end;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
user_uffdio_register = (struct uffdio_register __user *) arg;
*/
found = false;
basic_ioctls = false;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ mas_set(&mas, vma->vm_start);
+ mas_for_each(&mas, curr, end) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
+ vma = vma_next(vma);
} while (vma && vma->vm_start < end);
out_unlock:
mmap_write_unlock(mm);
bool found;
unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
*/
found = false;
ret = -EINVAL;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ mas_set(&mas, vma->vm_start);
+
+ mas_for_each(&mas, cur, end) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
+ vma = vma_next(vma);
} while (vma && vma->vm_start < end);
out_unlock:
mmap_write_unlock(mm);
extern struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
unsigned long start_addr, unsigned long end_addr);
+static inline struct vm_area_struct *vma_next(struct mm_struct *mm,
+ struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+
+ mas_set(&mas, vma->vm_end);
+ return mas_next(&mas, ULONG_MAX);
+}
+
+static inline struct vm_area_struct *vma_prev(struct mm_struct *mm,
+ struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+
+ mas_set(&mas, vma->vm_start);
+ return mas_prev(&mas, 0);
+}
+
static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
{
unsigned long vm_start = vma->vm_start;
return vma;
}
+static inline struct vm_area_struct *vma_mas_next(struct ma_state *mas)
+{
+ struct ma_state tmp;
+
+ memcpy(&tmp, mas, sizeof(tmp));
+ return mas_next(&tmp, ULONG_MAX);
+}
+
+static inline struct vm_area_struct *vma_mas_prev(struct ma_state *mas)
+{
+ struct ma_state tmp;
+
+ memcpy(&tmp, mas, sizeof(tmp));
+ return mas_prev(&tmp, 0);
+}
static inline bool range_in_vma(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
loff_t size = 0;
struct file *file;
struct vm_area_struct *next;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
#endif
if (addr & ~PAGE_MASK)
* match the usual checks anyway. So assume all vma's are
* above the starting address given.
*/
- vma = find_vma(mm, addr);
#ifdef CONFIG_MMU
+ vma = mas_find(&mas, ULONG_MAX);
while (vma) {
- next = vma->vm_next;
+ next = mas_find(&mas, ULONG_MAX);
/*
* Check if the starting address would match, i.e. it's
* prevent overflows and make comparisons vs. equal-width types.
*/
size = PAGE_ALIGN(size);
- while (vma && (loff_t)(vma->vm_end - addr) <= size) {
- next = vma->vm_next;
+ mas_for_each(&mas, vma, size) {
/* finding a matching vma now does not alter retval */
if ((vma->vm_ops == &shm_vm_ops) &&
((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
(vma->vm_file == file))
do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
- vma = next;
}
#else /* CONFIG_MMU */
+ vma = mas_walk(&mas);
/* under NOMMU conditions, the exact address to be destroyed must be
* given
*/
- if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
+ if (vma && vma->vm_ops == &shm_vm_ops) {
do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
retval = 0;
}
vma = current->mm->mmap;
while (vma) {
vsize += vma->vm_end - vma->vm_start;
- vma = vma->vm_next;
+ vma = vma_next(vma->vm_mm, vma);
}
mmap_read_unlock(current->mm);
}
struct perf_addr_filter_range *fr)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!vma->vm_file)
continue;
find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
{
struct vm_area_struct *tmp;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
+ mas_for_each(&mas, tmp, ULONG_MAX)
if (valid_ref_ctr_vma(uprobe, tmp))
return tmp;
{
struct vm_area_struct *vma;
int err = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
unsigned long vaddr;
loff_t offset;
static void mmf_recalc_uprobes(struct mm_struct *mm)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!valid_vma(vma, false))
continue;
/*
unsigned long start, end;
unsigned long nr_pte_updates = 0;
long pages, virtpages;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
if (!mmap_read_trylock(mm))
return;
- vma = find_vma(mm, start);
+
+ mas_set(&mas, start);
+ vma = mas_walk(&mas);
if (!vma) {
reset_ptenuma_scan(p);
+ mas_reset(&mas);
start = 0;
- vma = mm->mmap;
+ mas_set(&mas, start);
}
- for (; vma; vma = vma->vm_next) {
+
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
continue;
err = -EBUSY;
if (exe_file) {
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!vma->vm_file)
continue;
if (path_equal(&vma->vm_file->f_path,
struct vm_area_struct *vma = NULL;
int locked = 0;
long ret = 0;
+ MA_STATE(mas, &mm->mm_mt, start, start);
end = start + len;
if (!locked) {
locked = 1;
mmap_read_lock(mm);
- vma = find_vma(mm, nstart);
+ vma = mas_find(&mas, end);
} else if (nstart >= vma->vm_end)
- vma = vma->vm_next;
- if (!vma || vma->vm_start >= end)
+ vma = mas_next(&mas, end);
+ if (!vma)
break;
/*
* Set [nstart; nend) to intersection of desired address
split_huge_pmd_address(vma, end, false, NULL);
/*
- * If we're also updating the vma->vm_next->vm_start, if the new
- * vm_next->vm_start isn't hpage aligned and it could previously
+ * If we're also updating the vma_next(vma)->vm_start, if the new
+ * vma_next()->vm_start isn't hpage aligned and it could previously
* contain an hugepage: check if we need to split an huge pmd.
*/
if (adjust_next > 0) {
- struct vm_area_struct *next = vma->vm_next;
+ struct vm_area_struct *next = vma_next(vma);
unsigned long nstart = next->vm_start;
nstart += adjust_next;
if (nstart & ~HPAGE_PMD_MASK &&
struct mm_struct *mm;
struct vm_area_struct *vma;
int progress = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
VM_BUG_ON(!pages);
lockdep_assert_held(&khugepaged_mm_lock);
* the next mm on the list.
*/
vma = NULL;
+ mas_set(&mas, khugepaged_scan.address);
if (unlikely(!mmap_read_trylock(mm)))
goto breakouterloop_mmap_lock;
if (likely(!khugepaged_test_exit(mm)))
- vma = find_vma(mm, khugepaged_scan.address);
+ vma = mas_find(&mas, ULONG_MAX);
progress++;
- for (; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
unsigned long hstart, hend;
cond_resched();
struct mm_struct *mm;
struct vm_area_struct *vma;
int err = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
spin_lock(&ksm_mmlist_lock);
ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
mm = mm_slot->mm;
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (ksm_test_exit(mm))
break;
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
struct vm_area_struct *vma;
struct rmap_item *rmap_item;
int nid;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
if (list_empty(&ksm_mm_head.mm_list))
return NULL;
mmap_read_lock(mm);
if (ksm_test_exit(mm))
vma = NULL;
- else
- vma = find_vma(mm, ksm_scan.address);
+ else {
+ mas_set(&mas, ksm_scan.address);
+ vma = mas_find(&mas, ULONG_MAX);
+ }
- for (; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!(vma->vm_flags & VM_MERGEABLE))
continue;
if (ksm_scan.address < vma->vm_start)
if (start >= end)
goto out;
if (prev)
- vma = prev->vm_next;
+ vma = vma_next(mm, prev);
else /* madvise_remove dropped mmap_lock */
vma = find_vma(mm, start);
}
* Check if this is a VM_IO | VM_PFNMAP VMA, which
* we can access using slightly different code.
*/
- vma = find_vma(mm, addr);
- if (!vma || vma->vm_start > addr)
+ vma = find_vma_intersection(mm, addr, addr + 1);
+ if (!vma)
break;
if (vma->vm_ops && vma->vm_ops->access)
ret = vma->vm_ops->access(vma, addr, buf,
void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ mas_for_each(&mas, vma, ULONG_MAX)
mpol_rebind_policy(vma->vm_policy, new);
mmap_write_unlock(mm);
}
static int queue_pages_test_walk(unsigned long start, unsigned long end,
struct mm_walk *walk)
{
- struct vm_area_struct *vma = walk->vma;
+ struct vm_area_struct *next, *vma = walk->vma;
struct queue_pages *qp = walk->private;
unsigned long endvma = vma->vm_end;
unsigned long flags = qp->flags;
/* hole at head side of range */
return -EFAULT;
}
+ next = vma_next(vma->vm_mm, vma);
if (!(flags & MPOL_MF_DISCONTIG_OK) &&
((vma->vm_end < qp->end) &&
- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
+ (!next || vma->vm_end < next->vm_start)))
/* hole at middle or tail of range */
return -EFAULT;
pgoff_t pgoff;
unsigned long vmstart;
unsigned long vmend;
+ MA_STATE(mas, &mm->mm_mt, start, start);
- vma = find_vma(mm, start);
+ vma = mas_find(&mas, ULONG_MAX);
VM_BUG_ON(!vma);
- prev = vma->vm_prev;
+ prev = vma_mas_prev(&mas);
if (start > vma->vm_start)
prev = vma;
- for (; vma && vma->vm_start < end; prev = vma, vma = next) {
- next = vma->vm_next;
+ mas_for_each(&mas, vma, end) {
+ next = vma_next(mm, vma);
vmstart = max(start, vma->vm_start);
vmend = min(end, vma->vm_end);
if (mpol_equal(vma_policy(vma), new_pol))
- continue;
+ goto next;
pgoff = vma->vm_pgoff +
((vmstart - vma->vm_start) >> PAGE_SHIFT);
new_pol, vma->vm_userfaultfd_ctx);
if (prev) {
vma = prev;
- next = vma->vm_next;
+ next = vma_next(mm, vma);
if (mpol_equal(vma_policy(vma), new_pol))
continue;
/* vma_merge() joined vma && vma->next, case 8 */
err = vma_replace_policy(vma, new_pol);
if (err)
goto out;
+next:
+ prev = vma;
}
out:
{
struct vm_area_struct *vma;
unsigned long address;
+ MA_STATE(mas, ¤t->mm->mm_mt, start, start);
- vma = find_vma(current->mm, start);
- while (vma) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
address = page_address_in_vma(page, vma);
if (address != -EFAULT)
break;
- vma = vma->vm_next;
}
if (PageHuge(page)) {
unsigned long nstart, end, tmp;
struct vm_area_struct * vma, * prev;
int error;
+ MA_STATE(mas, ¤t->mm->mm_mt, start, start);
VM_BUG_ON(offset_in_page(start));
VM_BUG_ON(len != PAGE_ALIGN(len));
return -EINVAL;
if (end == start)
return 0;
- vma = find_vma(current->mm, start);
- if (!vma || vma->vm_start > start)
+ vma = mas_walk(&mas);
+ if (!vma)
return -ENOMEM;
- prev = vma->vm_prev;
+ prev = mas_prev(&mas, 0);
if (start > vma->vm_start)
prev = vma;
if (nstart >= end)
break;
- vma = prev->vm_next;
+ vma = vma_next(prev->vm_mm, prev);
if (!vma || vma->vm_start != nstart) {
error = -ENOMEM;
break;
{
struct vm_area_struct *vma;
unsigned long count = 0;
+ MA_STATE(mas, &mm->mm_mt, start, start);
if (mm == NULL)
mm = current->mm;
- vma = find_vma(mm, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (vma == NULL)
vma = mm->mmap;
- for (; vma ; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (start >= vma->vm_end)
continue;
if (start + len <= vma->vm_start)
to_add |= VM_LOCKONFAULT;
}
- for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
+ for (vma = current->mm->mmap; vma ; vma = vma_next(vma->vm_mm, prev)) {
vm_flags_t newflags;
newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
return existing ? true : false;
}
-/*
- * vma_next() - Get the next VMA.
- * @mm: The mm_struct.
- * @vma: The current vma.
- *
- * If @vma is NULL, return the first vma in the mm.
- *
- * Returns: The next VMA after @vma.
- */
-static inline struct vm_area_struct *vma_next(struct mm_struct *mm,
- struct vm_area_struct *vma)
-{
- if (!vma)
- return mm->mmap;
-
- return vma->vm_next;
-}
-
static unsigned long count_vma_pages_range(struct mm_struct *mm,
unsigned long addr, unsigned long end)
{
return 0;
}
+/*
+ * vma_next_wrap() - Get the next VMA of the first.
+ * @mm: The mm_struct.
+ * @vma: The current vma.
+ *
+ * If @vma is NULL, return the first vma in the mm.
+ *
+ * Returns: The next VMA after @vma.
+ */
+static inline struct vm_area_struct *vma_next_wrap(struct mm_struct *mm,
+ struct vm_area_struct *vma)
+{
+ if (!vma)
+ return mm->mmap;
+
+ return vma_next(mm, vma);
+}
+
/*
* Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
* whether that can be merged with its predecessor or its successor.
if (vm_flags & VM_SPECIAL)
return NULL;
- next = vma_next(mm, prev);
+ next = vma_next_wrap(mm, prev);
area = next;
if (area && area->vm_end == end) /* cases 6, 7, 8 */
next = next->vm_next;
const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
(prot & PROT_READ);
+ MA_STATE(mas, ¤t->mm->mm_mt, start, start);
start = untagged_addr(start);
if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
goto out;
- vma = find_vma(current->mm, start);
+ vma = mas_walk(&mas);
error = -ENOMEM;
if (!vma)
goto out;
- prev = vma->vm_prev;
+ prev = vma_prev(vma->vm_mm, vma);
if (unlikely(grows & PROT_GROWSDOWN)) {
if (vma->vm_start >= end)
goto out;
if (nstart >= end)
goto out;
- vma = prev->vm_next;
+ vma = mas_next(&mas, ULONG_MAX);
if (!vma || vma->vm_start != nstart) {
error = -ENOMEM;
goto out;
if (excess) {
vma->vm_flags |= VM_ACCOUNT;
if (split)
- vma->vm_next->vm_flags |= VM_ACCOUNT;
+ vma_next(mm, vma)->vm_flags |= VM_ACCOUNT;
}
return new_addr;
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{
unsigned long end = vma->vm_end + delta;
+ struct vm_area_struct *next;
if (end < vma->vm_end) /* overflow */
return 0;
- if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
+ next = vma_next(vma->vm_mm, vma);
+ if (next && next->vm_start < end) /* intersection */
return 0;
if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
0, MAP_FIXED) & ~PAGE_MASK)
error = 0;
goto out_unlock;
}
- vma = vma->vm_next;
+ vma = vma_next(mm, vma);
}
}
out_unlock:
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* check the cache first */
vma = vmacache_find(mm, addr);
/* trawl the list (there may be multiple mappings in which addr
* resides) */
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->vm_start > addr)
return NULL;
if (vma->vm_end > addr) {
{
struct vm_area_struct *vma;
unsigned long end = addr + len;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* check the cache first */
vma = vmacache_find_exact(mm, addr, end);
/* trawl the list (there may be multiple mappings in which addr
* resides) */
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->vm_start < addr)
continue;
if (vma->vm_start > addr)
return -EINVAL;
if (end == vma->vm_end)
goto erase_whole_vma;
- vma = vma->vm_next;
+ vma = vma_next(vma);
} while (vma);
return -EINVAL;
} else {
mm->total_vm = 0;
while ((vma = mm->mmap)) {
- mm->mmap = vma->vm_next;
+ mm->mmap = vma_next(vma);
delete_vma_from_mm(vma);
delete_vma(mm, vma);
cond_resched();
{
struct vm_area_struct *vma;
bool ret = true;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/*
* Tell all users of get_user/copy_from_user etc... that the content
*/
set_bit(MMF_UNSTABLE, &mm->flags);
- for (vma = mm->mmap ; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!can_madv_lru_vma(vma))
continue;
} else { /* inside vma */
walk.vma = vma;
next = min(end, vma->vm_end);
- vma = vma->vm_next;
+ vma = vma_next(mm, vma);;
err = walk_page_test(start, next, &walk);
if (err > 0) {
{
struct vm_area_struct *vma;
int ret = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->anon_vma) {
ret = unuse_vma(vma, type, frontswap,
fs_pages_to_unuse);
projects / users / jedix / linux-maple.git / commitdiff