mm: Remove vmacache

The maple tree is able to find a VMA quick enough to no longer need the
vma cache.  Remove the vmacache to reduce work in keeping it up to date
and code complexity.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/fs/exec.c b/fs/exec.c
index 547a2390baf54f4e285fb41d3488025f1954aed9..aa466921d6a9761395f7d1e009ec2fbe20b6428e 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -28,7 +28,6 @@
 #include <linux/file.h>
 #include <linux/fdtable.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/stat.h>
 #include <linux/fcntl.h>
 #include <linux/swap.h>
@@ -1019,8 +1018,6 @@ static int exec_mmap(struct mm_struct *mm)
        activate_mm(active_mm, mm);
        if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
                local_irq_enable();
-       tsk->mm->vmacache_seqnum = 0;
-       vmacache_flush(tsk);
        task_unlock(tsk);
        if (old_mm) {
                mmap_read_unlock(old_mm);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 217aa2705d5dfaa88e330e2e4606cbb54d9cbe1f..5161d8ea2d729fb1f99a5343735f8604648996f5 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/pagewalk.h>
-#include <linux/vmacache.h>
 #include <linux/hugetlb.h>
 #include <linux/huge_mm.h>
 #include <linux/mount.h>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index f8824e5a2156b0fb6b6475540f021aa2d73744ee..4e87ea66ca3c752ab06e6e01e077fcf523e39cdb 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -379,7 +379,6 @@ struct mm_struct {
        struct {
                struct vm_area_struct *mmap;            /* list of VMAs */
                struct maple_tree mm_mt;
-               u64 vmacache_seqnum;                   /* per-thread vmacache */
 #ifdef CONFIG_MMU
                unsigned long (*get_unmapped_area) (struct file *filp,
                                unsigned long addr, unsigned long len,

diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index c1bc6731125cbbeb99b2bbe0d4757fad6bf5031f..33c9fa4d4f66aaee5ddfa6d9f0918cee398834b0 100644 (file)
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -31,11 +31,6 @@
 #define VMACACHE_SIZE (1U << VMACACHE_BITS)
 #define VMACACHE_MASK (VMACACHE_SIZE - 1)
 
-struct vmacache {
-       u64 seqnum;
-       struct vm_area_struct *vmas[VMACACHE_SIZE];
-};
-
 /*
  * When updating this, please also update struct resident_page_types[] in
  * kernel/fork.c

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 063cd120b459339443db7b003df5cd22140316c6..6e4cb78e5d1abf342822f2036d11af1a5bf47106 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -750,7 +750,6 @@ struct task_struct {
        struct mm_struct                *active_mm;
 
        /* Per-thread vma caching: */
-       struct vmacache                 vmacache;
 
 #ifdef SPLIT_RSS_COUNTING
        struct task_rss_stat            rss_stat;

diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h
deleted file mode 100644 (file)
index 6fce268..0000000
--- a/include/linux/vmacache.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_VMACACHE_H
-#define __LINUX_VMACACHE_H
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-
-static inline void vmacache_flush(struct task_struct *tsk)
-{
-       memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
-}
-
-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma);
-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm,
-                                                   unsigned long addr);
-
-#ifndef CONFIG_MMU
-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                                 unsigned long start,
-                                                 unsigned long end);
-#endif
-
-static inline void vmacache_invalidate(struct mm_struct *mm)
-{
-       mm->vmacache_seqnum++;
-}
-
-#endif /* __LINUX_VMACACHE_H */

diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 1e75a8923a8d1e14ab0a78743ec6210b3020b2cc..f6e7156088ea70460a45cbefc52abf5b131f6b36 100644 (file)
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -53,7 +53,6 @@
 #include <linux/pid.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/rcupdate.h>
 #include <linux/irq.h>
 
@@ -286,17 +285,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)
        if (!CACHE_FLUSH_IS_SAFE)
                return;
 
-       if (current->mm) {
-               int i;
-
-               for (i = 0; i < VMACACHE_SIZE; i++) {
-                       if (!current->vmacache.vmas[i])
-                               continue;
-                       flush_cache_range(current->vmacache.vmas[i],
-                                         addr, addr + BREAK_INSTR_SIZE);
-               }
-       }
-
        /* Force flush instruction cache if it was outside the mm */
        flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
 }

diff --git a/kernel/fork.c b/kernel/fork.c
index 5e4de4b860439540d4e8a01325d8dee1359636ad..52ff974dac5115938cf4d00caea92035f8f9558c 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -42,7 +42,6 @@
 #include <linux/mmu_notifier.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
@@ -1011,7 +1010,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 {
        mm->mmap = NULL;
        mt_init_flags(&mm->mm_mt, MAPLE_ALLOC_RANGE);
-       mm->vmacache_seqnum = 0;
        atomic_set(&mm->mm_users, 1);
        atomic_set(&mm->mm_count, 1);
        mmap_init_lock(mm);
@@ -1408,9 +1406,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
        if (!oldmm)
                return 0;
 
-       /* initialize the new vmacache entries */
-       vmacache_flush(tsk);
-
        if (clone_flags & CLONE_VM) {
                mmget(oldmm);
                mm = oldmm;

diff --git a/mm/Makefile b/mm/Makefile
index d73aed0fc99c1d408090c8175f482bbd24a6f2a9..ca05b84ac738bc2950e5cc385505ed2ba54ca5e9 100644 (file)
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -50,7 +50,7 @@ obj-y                 := filemap.o mempool.o oom_kill.o fadvise.o \
                           readahead.o swap.o truncate.o vmscan.o shmem.o \
                           util.o mmzone.o vmstat.o backing-dev.o \
                           mm_init.o percpu.o slab_common.o \
-                          compaction.o vmacache.o \
+                          compaction.o \
                           interval_tree.o list_lru.o workingset.o \
                           debug.o gup.o $(mmu-y)
 

diff --git a/mm/debug.c b/mm/debug.c
index ccca576b28990476dfdc1f73d44042a34d23bf3f..d8ed9d73832671f2970fb3e7edbf161add40a854 100644 (file)
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -214,7 +214,7 @@ EXPORT_SYMBOL(dump_vma);
 
 void dump_mm(const struct mm_struct *mm)
 {
-       pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n"
+       pr_emerg("mm %px mmap %px task_size %lu\n"
 #ifdef CONFIG_MMU
                "get_unmapped_area %px\n"
 #endif
@@ -242,7 +242,7 @@ void dump_mm(const struct mm_struct *mm)
                "tlb_flush_pending %d\n"
                "def_flags: %#lx(%pGv)\n",
 
-               mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size,
+               mm, mm->mmap, mm->task_size,
 #ifdef CONFIG_MMU
                mm->get_unmapped_area,
 #endif

diff --git a/mm/mmap.c b/mm/mmap.c
index 50672633f6ed295bb9d6df01c01d5890ffae88a5..de8838a4f8c6948fe1b6fb713c3f2610099103c6 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -13,7 +13,6 @@
 #include <linux/slab.h>
 #include <linux/backing-dev.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/shm.h>
 #include <linux/mman.h>
 #include <linux/pagemap.h>
@@ -697,9 +696,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
                /* Remove from mm linked list - also updates highest_vm_end */
                __vma_unlink_list(mm, next);
 
-               /* Kill the cache */
-               vmacache_invalidate(mm);
-
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
 
@@ -915,8 +911,6 @@ again:
 
        if (remove_next) {
                __vma_unlink_list(mm, next);
-               /* Kill the cache */
-               vmacache_invalidate(mm);
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
        } else if (insert) {
@@ -2201,19 +2195,9 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
 {
        struct vm_area_struct *vma;
        MA_STATE(mas, &mm->mm_mt, start_addr, start_addr);
-
-       /* Check the cache first. */
-       vma = vmacache_find(mm, start_addr);
-       if (likely(vma))
-               return vma;
-
        rcu_read_lock();
        vma = mas_find(&mas, end_addr - 1);
        rcu_read_unlock();
-
-       if (vma)
-               vmacache_update(start_addr, vma);
-
        return vma;
 }
 EXPORT_SYMBOL(find_vma_intersection);
@@ -2608,9 +2592,6 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
        tail_vma->vm_next = NULL;
 
-       /* Kill the cache */
-       vmacache_invalidate(mm);
-
        /*
         * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
         * VM_GROWSUP VMA. Such VMAs can change their size under
@@ -2999,7 +2980,6 @@ static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
        if (vma_mas_remove(&unmap, mas))
                goto mas_store_fail;
 
-       vmacache_invalidate(mm);
        unmap_pages = vma_pages(&unmap);
        if (unmap.vm_flags & VM_LOCKED) {
                mm->locked_vm -= unmap_pages;

diff --git a/mm/vmacache.c b/mm/vmacache.c
deleted file mode 100644 (file)
index 01a6e66..0000000
--- a/mm/vmacache.c
+++ /dev/null
@@ -1,117 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2014 Davidlohr Bueso.
- */
-#include <linux/sched/signal.h>
-#include <linux/sched/task.h>
-#include <linux/mm.h>
-#include <linux/vmacache.h>
-
-/*
- * Hash based on the pmd of addr if configured with MMU, which provides a good
- * hit rate for workloads with spatial locality.  Otherwise, use pages.
- */
-#ifdef CONFIG_MMU
-#define VMACACHE_SHIFT PMD_SHIFT
-#else
-#define VMACACHE_SHIFT PAGE_SHIFT
-#endif
-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
-
-/*
- * This task may be accessing a foreign mm via (for example)
- * get_user_pages()->find_vma().  The vmacache is task-local and this
- * task's vmacache pertains to a different mm (ie, its own).  There is
- * nothing we can do here.
- *
- * Also handle the case where a kernel thread has adopted this mm via
- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
- */
-static inline bool vmacache_valid_mm(struct mm_struct *mm)
-{
-       return current->mm == mm && !(current->flags & PF_KTHREAD);
-}
-
-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
-{
-       if (vmacache_valid_mm(newvma->vm_mm))
-               current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
-}
-
-static bool vmacache_valid(struct mm_struct *mm)
-{
-       struct task_struct *curr;
-
-       if (!vmacache_valid_mm(mm))
-               return false;
-
-       curr = current;
-       if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
-               /*
-                * First attempt will always be invalid, initialize
-                * the new cache for this task here.
-                */
-               curr->vmacache.seqnum = mm->vmacache_seqnum;
-               vmacache_flush(curr);
-               return false;
-       }
-       return true;
-}
-
-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
-{
-       int idx = VMACACHE_HASH(addr);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma) {
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-                       if (WARN_ON_ONCE(vma->vm_mm != mm))
-                               break;
-#endif
-                       if (vma->vm_start <= addr && vma->vm_end > addr) {
-                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                               return vma;
-                       }
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-
-#ifndef CONFIG_MMU
-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                          unsigned long start,
-                                          unsigned long end)
-{
-       int idx = VMACACHE_HASH(start);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma && vma->vm_start == start && vma->vm_end == end) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-#endif