mm: Remove vmacache

By using the maple tree and the maple tree state, the vmacache is no
longer beneficial and is complicating the VMA code.  Remove the vmacache
to reduce the work in keeping it up to date and code complexity.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 6aad18ee131d6634efd7188b9660d723f2ea36a6..2d05af1a7a3ec28e495eb20bf7ad2b74b64310f8 100644 (file)
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -801,7 +801,6 @@ CONFIG_SLUB_DEBUG_ON=y
 CONFIG_SLUB_STATS=y
 CONFIG_DEBUG_STACK_USAGE=y
 CONFIG_DEBUG_VM=y
-CONFIG_DEBUG_VM_VMACACHE=y
 CONFIG_DEBUG_VM_PGFLAGS=y
 CONFIG_DEBUG_MEMORY_INIT=y
 CONFIG_MEMORY_NOTIFIER_ERROR_INJECT=m

diff --git a/fs/exec.c b/fs/exec.c
index a098c133d8d74039e46256ac365883148911b9ae..c618a55a20d3fe57f1fbc86767fa57742b9d9425 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -28,7 +28,6 @@
 #include <linux/file.h>
 #include <linux/fdtable.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/stat.h>
 #include <linux/fcntl.h>
 #include <linux/swap.h>
@@ -1019,8 +1018,6 @@ static int exec_mmap(struct mm_struct *mm)
        activate_mm(active_mm, mm);
        if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
                local_irq_enable();
-       tsk->mm->vmacache_seqnum = 0;
-       vmacache_flush(tsk);
        task_unlock(tsk);
        if (old_mm) {
                mmap_read_unlock(old_mm);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index cf25be3e0321206b5a45f66ae2cfc2084e7f947c..63d07414d583637658e3436128ceb6c9674f0950 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/pagewalk.h>
-#include <linux/vmacache.h>
 #include <linux/hugetlb.h>
 #include <linux/huge_mm.h>
 #include <linux/mount.h>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index d49a9064be782bcce9339138ecf0b0c8ed7826d0..47250fcca5ecd0dec19fb7d0e02015c2809d2386 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -392,7 +392,6 @@ struct mm_struct {
        struct {
                struct vm_area_struct *mmap;            /* list of VMAs */
                struct maple_tree mm_mt;
-               u64 vmacache_seqnum;                   /* per-thread vmacache */
 #ifdef CONFIG_MMU
                unsigned long (*get_unmapped_area) (struct file *filp,
                                unsigned long addr, unsigned long len,

diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index c1bc6731125cbbeb99b2bbe0d4757fad6bf5031f..33c9fa4d4f66aaee5ddfa6d9f0918cee398834b0 100644 (file)
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -31,11 +31,6 @@
 #define VMACACHE_SIZE (1U << VMACACHE_BITS)
 #define VMACACHE_MASK (VMACACHE_SIZE - 1)
 
-struct vmacache {
-       u64 seqnum;
-       struct vm_area_struct *vmas[VMACACHE_SIZE];
-};
-
 /*
  * When updating this, please also update struct resident_page_types[] in
  * kernel/fork.c

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 39039ce8ac4c1774a8c1aca1d0d7bc5eb6e7485f..d9f93aab65512951a0cf13fbd0cb8f4cc8f859f6 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -858,7 +858,6 @@ struct task_struct {
        struct mm_struct                *active_mm;
 
        /* Per-thread vma caching: */
-       struct vmacache                 vmacache;
 
 #ifdef SPLIT_RSS_COUNTING
        struct task_rss_stat            rss_stat;

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index a185cc75ff52ae4d56fe0e92ab194e645e5ab9b3..dd9e1e98bdec9614d01c7f1c316b79376db0a060 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -119,10 +119,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                NR_TLB_LOCAL_FLUSH_ALL,
                NR_TLB_LOCAL_FLUSH_ONE,
 #endif /* CONFIG_DEBUG_TLBFLUSH */
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-               VMACACHE_FIND_CALLS,
-               VMACACHE_FIND_HITS,
-#endif
 #ifdef CONFIG_SWAP
                SWAP_RA,
                SWAP_RA_HIT,

diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h
deleted file mode 100644 (file)
index 6fce268..0000000
--- a/include/linux/vmacache.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_VMACACHE_H
-#define __LINUX_VMACACHE_H
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-
-static inline void vmacache_flush(struct task_struct *tsk)
-{
-       memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
-}
-
-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma);
-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm,
-                                                   unsigned long addr);
-
-#ifndef CONFIG_MMU
-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                                 unsigned long start,
-                                                 unsigned long end);
-#endif
-
-static inline void vmacache_invalidate(struct mm_struct *mm)
-{
-       mm->vmacache_seqnum++;
-}
-
-#endif /* __LINUX_VMACACHE_H */

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index d6a6cf53b1278f40bfe6cdeab9c7e30b75f406fc..f9a2924a217454a7b8c9d1dfecb3d6e3083bb1b1 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu)
 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
 #endif
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-#define count_vm_vmacache_event(x) count_vm_event(x)
-#else
-#define count_vm_vmacache_event(x) do {} while (0)
-#endif
-
 #define __count_zid_vm_events(item, zid, delta) \
        __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
 

diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index da06a5553835b5e443555ee6b446e93abc2c3ffe..c4e6f5159bedb64878af7562f21eaef4e55b8153 100644 (file)
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -50,7 +50,6 @@
 #include <linux/pid.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/rcupdate.h>
 #include <linux/irq.h>
 
@@ -282,17 +281,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)
        if (!CACHE_FLUSH_IS_SAFE)
                return;
 
-       if (current->mm) {
-               int i;
-
-               for (i = 0; i < VMACACHE_SIZE; i++) {
-                       if (!current->vmacache.vmas[i])
-                               continue;
-                       flush_cache_range(current->vmacache.vmas[i],
-                                         addr, addr + BREAK_INSTR_SIZE);
-               }
-       }
-
        /* Force flush instruction cache if it was outside the mm */
        flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
 }

diff --git a/kernel/fork.c b/kernel/fork.c
index 86b6beb805000e5425146211861186fc8002589a..31c349702746241a655126d072dae2bc33675c0c 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -42,7 +42,6 @@
 #include <linux/mmu_notifier.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
@@ -1048,7 +1047,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
        mm->mmap = NULL;
        mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
        mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
-       mm->vmacache_seqnum = 0;
        atomic_set(&mm->mm_users, 1);
        atomic_set(&mm->mm_count, 1);
        seqcount_init(&mm->write_protect_seq);
@@ -1506,9 +1504,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
        if (!oldmm)
                return 0;
 
-       /* initialize the new vmacache entries */
-       vmacache_flush(tsk);
-
        if (clone_flags & CLONE_VM) {
                mmget(oldmm);
                mm = oldmm;

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 6748fe0fffd8880bc3ff46602466fdc6c7a52265..74e8836a4e8b19d55c45d2aec82cc601c6d767ce 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -802,16 +802,6 @@ config DEBUG_VM
 
          If unsure, say N.
 
-config DEBUG_VM_VMACACHE
-       bool "Debug VMA caching"
-       depends on DEBUG_VM
-       help
-         Enable this to turn on VMA caching debug information. Doing so
-         can cause significant overhead, so only enable it in non-production
-         environments.
-
-         If unsure, say N.
-
 config DEBUG_VM_RB
        bool "Debug VM red-black trees"
        depends on DEBUG_VM

diff --git a/mm/Makefile b/mm/Makefile
index fc60a40ce954bd911005e2c178dcbb4590bfe29c..fed159b876a984018b15d4af9e7563c331b62de8 100644 (file)
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -50,7 +50,7 @@ obj-y                 := filemap.o mempool.o oom_kill.o fadvise.o \
                           readahead.o swap.o truncate.o vmscan.o shmem.o \
                           util.o mmzone.o vmstat.o backing-dev.o \
                           mm_init.o percpu.o slab_common.o \
-                          compaction.o vmacache.o \
+                          compaction.o \
                           interval_tree.o list_lru.o workingset.o \
                           debug.o gup.o mmap_lock.o $(mmu-y)
 

diff --git a/mm/debug.c b/mm/debug.c
index fae0f81ad831302462b99257602c067359668ab6..5fe665c6ce1c6361dc39c3278601d950ee0044f0 100644 (file)
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -205,7 +205,7 @@ EXPORT_SYMBOL(dump_vma);
 
 void dump_mm(const struct mm_struct *mm)
 {
-       pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n"
+       pr_emerg("mm %px mmap %px task_size %lu\n"
 #ifdef CONFIG_MMU
                "get_unmapped_area %px\n"
 #endif
@@ -233,7 +233,7 @@ void dump_mm(const struct mm_struct *mm)
                "tlb_flush_pending %d\n"
                "def_flags: %#lx(%pGv)\n",
 
-               mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size,
+               mm, mm->mmap, mm->task_size,
 #ifdef CONFIG_MMU
                mm->get_unmapped_area,
 #endif

diff --git a/mm/mmap.c b/mm/mmap.c
index 47bec4671487e734546b7ab22145d87e91320973..f75caaa8f006dcb15c577d3b71f8bd216f530dad 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -13,7 +13,6 @@
 #include <linux/slab.h>
 #include <linux/backing-dev.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/shm.h>
 #include <linux/mman.h>
 #include <linux/pagemap.h>
@@ -676,9 +675,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
                /* Remove from mm linked list - also updates highest_vm_end */
                __vma_unlink_list(mm, next);
 
-               /* Kill the cache */
-               vmacache_invalidate(mm);
-
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
 
@@ -898,8 +894,6 @@ again:
 
        if (remove_next) {
                __vma_unlink_list(mm, next);
-               /* Kill the cache */
-               vmacache_invalidate(mm);
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
        } else if (insert) {
@@ -2176,16 +2170,9 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
        MA_STATE(mas, &mm->mm_mt, start_addr, start_addr);
 
        mmap_assert_locked(mm);
-       /* Check the cache first. */
-       vma = vmacache_find(mm, start_addr);
-       if (likely(vma))
-               return vma;
-
        rcu_read_lock();
        vma = mas_find(&mas, end_addr - 1);
        rcu_read_unlock();
-       if (vma)
-               vmacache_update(mas.index, vma);
 
        return vma;
 }
@@ -2576,9 +2563,6 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
        tail_vma->vm_next = NULL;
 
-       /* Kill the cache */
-       vmacache_invalidate(mm);
-
        /*
         * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
         * VM_GROWSUP VMA. Such VMAs can change their size under
@@ -2958,7 +2942,6 @@ static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
        if (vma_mas_remove(&unmap, mas))
                goto mas_store_fail;
 
-       vmacache_invalidate(vma->vm_mm);
        if (vma->anon_vma) {
                anon_vma_interval_tree_post_update_vma(vma);
                anon_vma_unlock_write(vma->anon_vma);

diff --git a/mm/nommu.c b/mm/nommu.c
index 86b162011f7689939e75efe4b541fceb6615a6e8..9833f0f7710366b99f58ab6a58e6b38bbb5d87b7 100644 (file)
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -19,7 +19,6 @@
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/sched/mm.h>
-#include <linux/vmacache.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/file.h>
@@ -586,22 +585,12 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
  */
 static void delete_vma_from_mm(struct vm_area_struct *vma)
 {
-       int i;
-       struct address_space *mapping;
-       struct mm_struct *mm = vma->vm_mm;
-       struct task_struct *curr = current;
        MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
 
-       mm->map_count--;
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               /* if the vma is cached, invalidate the entire cache */
-               if (curr->vmacache.vmas[i] == vma) {
-                       vmacache_invalidate(mm);
-                       break;
-               }
-       }
+       vma->vm_mm->map_count--;
        /* remove the VMA from the mapping */
        if (vma->vm_file) {
+               struct address_space *mapping;
                mapping = vma->vm_file->f_mapping;
 
                i_mmap_lock_write(mapping);
@@ -613,7 +602,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
 
        /* remove from the MM's tree and list */
        vma_mas_remove(vma, &mas);
-       __vma_unlink_list(mm, vma);
+       __vma_unlink_list(vma->vm_mm, vma);
 }
 
 /*
@@ -638,18 +627,10 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
        struct vm_area_struct *vma;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find(mm, addr);
-       if (likely(vma))
-               return vma;
-
        rcu_read_lock();
        vma = mas_walk(&mas);
        rcu_read_unlock();
 
-       if (vma)
-               vmacache_update(addr, vma);
-
        return vma;
 }
 EXPORT_SYMBOL(find_vma);
@@ -684,11 +665,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        unsigned long end = addr + len;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find_exact(mm, addr, end);
-       if (vma)
-               return vma;
-
        rcu_read_lock();
        vma = mas_walk(&mas);
        rcu_read_unlock();
@@ -699,7 +675,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        if (vma->vm_end != end)
                return NULL;
 
-       vmacache_update(addr, vma);
        return vma;
 }
 

diff --git a/mm/vmacache.c b/mm/vmacache.c
deleted file mode 100644 (file)
index 01a6e66..0000000
--- a/mm/vmacache.c
+++ /dev/null
@@ -1,117 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2014 Davidlohr Bueso.
- */
-#include <linux/sched/signal.h>
-#include <linux/sched/task.h>
-#include <linux/mm.h>
-#include <linux/vmacache.h>
-
-/*
- * Hash based on the pmd of addr if configured with MMU, which provides a good
- * hit rate for workloads with spatial locality.  Otherwise, use pages.
- */
-#ifdef CONFIG_MMU
-#define VMACACHE_SHIFT PMD_SHIFT
-#else
-#define VMACACHE_SHIFT PAGE_SHIFT
-#endif
-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
-
-/*
- * This task may be accessing a foreign mm via (for example)
- * get_user_pages()->find_vma().  The vmacache is task-local and this
- * task's vmacache pertains to a different mm (ie, its own).  There is
- * nothing we can do here.
- *
- * Also handle the case where a kernel thread has adopted this mm via
- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
- */
-static inline bool vmacache_valid_mm(struct mm_struct *mm)
-{
-       return current->mm == mm && !(current->flags & PF_KTHREAD);
-}
-
-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
-{
-       if (vmacache_valid_mm(newvma->vm_mm))
-               current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
-}
-
-static bool vmacache_valid(struct mm_struct *mm)
-{
-       struct task_struct *curr;
-
-       if (!vmacache_valid_mm(mm))
-               return false;
-
-       curr = current;
-       if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
-               /*
-                * First attempt will always be invalid, initialize
-                * the new cache for this task here.
-                */
-               curr->vmacache.seqnum = mm->vmacache_seqnum;
-               vmacache_flush(curr);
-               return false;
-       }
-       return true;
-}
-
-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
-{
-       int idx = VMACACHE_HASH(addr);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma) {
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-                       if (WARN_ON_ONCE(vma->vm_mm != mm))
-                               break;
-#endif
-                       if (vma->vm_start <= addr && vma->vm_end > addr) {
-                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                               return vma;
-                       }
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-
-#ifndef CONFIG_MMU
-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                          unsigned long start,
-                                          unsigned long end)
-{
-       int idx = VMACACHE_HASH(start);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma && vma->vm_start == start && vma->vm_end == end) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-#endif

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 8ce2620344b2f8b4e673c8f48e92945b874ef36d..25683a0e0b953fec69b5a1ed18d1de43328676ef 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1369,10 +1369,6 @@ const char * const vmstat_text[] = {
        "nr_tlb_local_flush_one",
 #endif /* CONFIG_DEBUG_TLBFLUSH */
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-       "vmacache_find_calls",
-       "vmacache_find_hits",
-#endif
 #ifdef CONFIG_SWAP
        "swap_ra",
        "swap_ra_hit",