mm: Remove vmacache

By using the maple tree and the maple tree state, the vmacache is no
longer beneficial and is complicating the VMA code.  Remove the vmacache
to reduce the work in keeping it up to date and code complexity.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 86afcc6b56bf745331f0669ce99a0d1c5ed2c477..f754bf7112619ccf0598c2b829236bdcd13d5dca 100644 (file)
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -791,7 +791,6 @@ CONFIG_SLUB_DEBUG_ON=y
 CONFIG_SLUB_STATS=y
 CONFIG_DEBUG_STACK_USAGE=y
 CONFIG_DEBUG_VM=y
-CONFIG_DEBUG_VM_VMACACHE=y
 CONFIG_DEBUG_VM_PGFLAGS=y
 CONFIG_DEBUG_MEMORY_INIT=y
 CONFIG_MEMORY_NOTIFIER_ERROR_INJECT=m

diff --git a/fs/exec.c b/fs/exec.c
index 38f63451b92827cef1aebaf948848d286b04900a..3dfa1e6d237f529e80228885007c10882e074309 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -28,7 +28,6 @@
 #include <linux/file.h>
 #include <linux/fdtable.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/stat.h>
 #include <linux/fcntl.h>
 #include <linux/swap.h>
@@ -1017,8 +1016,6 @@ static int exec_mmap(struct mm_struct *mm)
        activate_mm(active_mm, mm);
        if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
                local_irq_enable();
-       tsk->mm->vmacache_seqnum = 0;
-       vmacache_flush(tsk);
        task_unlock(tsk);
        if (old_mm) {
                mmap_read_unlock(old_mm);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index eb97468dfe4ca0c63234a0f576f954811db1371a..ab9b4ff44b54234d62ec2ee1bf0995eeb7a32ce9 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/pagewalk.h>
-#include <linux/vmacache.h>
 #include <linux/hugetlb.h>
 #include <linux/huge_mm.h>
 #include <linux/mount.h>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 55635e04376f3b1e9b9c8ef274db8f0872ba05d4..15304a3deb28a034002b6b722be9e84a9dfb0774 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -380,7 +380,6 @@ struct mm_struct {
        struct {
                struct vm_area_struct *mmap;            /* list of VMAs */
                struct maple_tree mm_mt;
-               u64 vmacache_seqnum;                   /* per-thread vmacache */
 #ifdef CONFIG_MMU
                unsigned long (*get_unmapped_area) (struct file *filp,
                                unsigned long addr, unsigned long len,

diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index 1b222f8039d1088db4bb3e375d4f8adc8f673a26..418ec7df5f188b7c4d4ad2a7d8c9a7cedeae0c12 100644 (file)
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -36,11 +36,6 @@
 #define VMACACHE_SIZE (1U << VMACACHE_BITS)
 #define VMACACHE_MASK (VMACACHE_SIZE - 1)
 
-struct vmacache {
-       u64 seqnum;
-       struct vm_area_struct *vmas[VMACACHE_SIZE];
-};
-
 /*
  * When updating this, please also update struct resident_page_types[] in
  * kernel/fork.c

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 38145174c7b1fb5a3bada932332d2d4a2f3f4e7b..5b4e2c888353521004757a24ad1f0c0e5ef7e81f 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -779,7 +779,6 @@ struct task_struct {
        struct mm_struct                *active_mm;
 
        /* Per-thread vma caching: */
-       struct vmacache                 vmacache;
 
 #ifdef SPLIT_RSS_COUNTING
        struct task_rss_stat            rss_stat;

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index ae0dd1948c2b0521dd8461f5d8828b7f6d25f7ed..cd3ff075470b96964c0d99025c3bafef8b51c1c4 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -117,10 +117,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                NR_TLB_LOCAL_FLUSH_ALL,
                NR_TLB_LOCAL_FLUSH_ONE,
 #endif /* CONFIG_DEBUG_TLBFLUSH */
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-               VMACACHE_FIND_CALLS,
-               VMACACHE_FIND_HITS,
-#endif
 #ifdef CONFIG_SWAP
                SWAP_RA,
                SWAP_RA_HIT,

diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h
deleted file mode 100644 (file)
index 6fce268..0000000
--- a/include/linux/vmacache.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_VMACACHE_H
-#define __LINUX_VMACACHE_H
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-
-static inline void vmacache_flush(struct task_struct *tsk)
-{
-       memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
-}
-
-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma);
-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm,
-                                                   unsigned long addr);
-
-#ifndef CONFIG_MMU
-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                                 unsigned long start,
-                                                 unsigned long end);
-#endif
-
-static inline void vmacache_invalidate(struct mm_struct *mm)
-{
-       mm->vmacache_seqnum++;
-}
-
-#endif /* __LINUX_VMACACHE_H */

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index d6a6cf53b1278f40bfe6cdeab9c7e30b75f406fc..f9a2924a217454a7b8c9d1dfecb3d6e3083bb1b1 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu)
 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
 #endif
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-#define count_vm_vmacache_event(x) count_vm_event(x)
-#else
-#define count_vm_vmacache_event(x) do {} while (0)
-#endif
-
 #define __count_zid_vm_events(item, zid, delta) \
        __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
 

diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 4708aec492df9e54145bfc4084adeb4b21639f5e..1bd0bb76ed2cebad5a27ab32745f4fde67d511f5 100644 (file)
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -53,7 +53,6 @@
 #include <linux/pid.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/rcupdate.h>
 #include <linux/irq.h>
 
@@ -285,17 +284,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)
        if (!CACHE_FLUSH_IS_SAFE)
                return;
 
-       if (current->mm) {
-               int i;
-
-               for (i = 0; i < VMACACHE_SIZE; i++) {
-                       if (!current->vmacache.vmas[i])
-                               continue;
-                       flush_cache_range(current->vmacache.vmas[i],
-                                         addr, addr + BREAK_INSTR_SIZE);
-               }
-       }
-
        /* Force flush instruction cache if it was outside the mm */
        flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
 }

diff --git a/kernel/fork.c b/kernel/fork.c
index 08db38b8984cd8b8f13ad5adda4b781b608ef63f..ab5f47754ca66e6c01e3f9ec1b2990bd36b6d456 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -42,7 +42,6 @@
 #include <linux/mmu_notifier.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
@@ -1042,7 +1041,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 {
        mm->mmap = NULL;
        mt_init_flags(&mm->mm_mt, MAPLE_ALLOC_RANGE);
-       mm->vmacache_seqnum = 0;
        atomic_set(&mm->mm_users, 1);
        atomic_set(&mm->mm_count, 1);
        seqcount_init(&mm->write_protect_seq);
@@ -1439,9 +1437,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
        if (!oldmm)
                return 0;
 
-       /* initialize the new vmacache entries */
-       vmacache_flush(tsk);
-
        if (clone_flags & CLONE_VM) {
                mmget(oldmm);
                mm = oldmm;

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 863cb6a62c175eb74e7a29dd8513d2c92eed7ea9..c54365c535786fb5590ef9a22cb1887d3eec7ad8 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -803,16 +803,6 @@ config DEBUG_VM
 
          If unsure, say N.
 
-config DEBUG_VM_VMACACHE
-       bool "Debug VMA caching"
-       depends on DEBUG_VM
-       help
-         Enable this to turn on VMA caching debug information. Doing so
-         can cause significant overhead, so only enable it in non-production
-         environments.
-
-         If unsure, say N.
-
 config DEBUG_VM_RB
        bool "Debug VM red-black trees"
        depends on DEBUG_VM

diff --git a/mm/Makefile b/mm/Makefile
index e3436741d5391c98d594a9ae1638073a68375c79..0cd70dcd75a924ab0bfef6082063784225a23676 100644 (file)
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -50,7 +50,7 @@ obj-y                 := filemap.o mempool.o oom_kill.o fadvise.o \
                           readahead.o swap.o truncate.o vmscan.o shmem.o \
                           util.o mmzone.o vmstat.o backing-dev.o \
                           mm_init.o percpu.o slab_common.o \
-                          compaction.o vmacache.o \
+                          compaction.o \
                           interval_tree.o list_lru.o workingset.o \
                           debug.o gup.o mmap_lock.o $(mmu-y)
 

diff --git a/mm/debug.c b/mm/debug.c
index e73fe0a8ec3d25fe4af305cb89c83288fbba1d18..f84662c87b4fc97d24e07703556fc49481c399ba 100644 (file)
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -203,7 +203,7 @@ EXPORT_SYMBOL(dump_vma);
 
 void dump_mm(const struct mm_struct *mm)
 {
-       pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n"
+       pr_emerg("mm %px mmap %px task_size %lu\n"
 #ifdef CONFIG_MMU
                "get_unmapped_area %px\n"
 #endif
@@ -231,7 +231,7 @@ void dump_mm(const struct mm_struct *mm)
                "tlb_flush_pending %d\n"
                "def_flags: %#lx(%pGv)\n",
 
-               mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size,
+               mm, mm->mmap, mm->task_size,
 #ifdef CONFIG_MMU
                mm->get_unmapped_area,
 #endif

diff --git a/mm/mmap.c b/mm/mmap.c
index f9209d9145bedfab01030e30bffaf32278518396..48fd97f3d7aec87fdc0ebd7e96f4a831eef76806 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -13,7 +13,6 @@
 #include <linux/slab.h>
 #include <linux/backing-dev.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/shm.h>
 #include <linux/mman.h>
 #include <linux/pagemap.h>
@@ -712,9 +711,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
                /* Remove from mm linked list - also updates highest_vm_end */
                __vma_unlink_list(mm, next);
 
-               /* Kill the cache */
-               vmacache_invalidate(mm);
-
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
 
@@ -929,8 +925,6 @@ again:
 
        if (remove_next) {
                __vma_unlink_list(mm, next);
-               /* Kill the cache */
-               vmacache_invalidate(mm);
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
        } else if (insert) {
@@ -2225,16 +2219,9 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
        struct vm_area_struct *vma;
        MA_STATE(mas, &mm->mm_mt, start_addr, start_addr);
 
-       /* Check the cache first. */
-       vma = vmacache_find(mm, start_addr);
-       if (likely(vma))
-               return vma;
-
        rcu_read_lock();
        vma = mas_find(&mas, end_addr - 1);
        rcu_read_unlock();
-       if (vma)
-               vmacache_update(mas.index, vma);
 
        return vma;
 }
@@ -2627,9 +2614,6 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
        tail_vma->vm_next = NULL;
 
-       /* Kill the cache */
-       vmacache_invalidate(mm);
-
        /*
         * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
         * VM_GROWSUP VMA. Such VMAs can change their size under
@@ -3009,7 +2993,6 @@ static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
        if (vma_mas_remove(&unmap, mas))
                goto mas_store_fail;
 
-       vmacache_invalidate(vma->vm_mm);
        if (vma->anon_vma) {
                anon_vma_interval_tree_post_update_vma(vma);
                anon_vma_unlock_write(vma->anon_vma);

diff --git a/mm/nommu.c b/mm/nommu.c
index eac5ca450bef573395eeb4913680660f07d86ac7..675aee78f8f2e81e061592a5996fddc2828afcc0 100644 (file)
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -19,7 +19,6 @@
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/sched/mm.h>
-#include <linux/vmacache.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/file.h>
@@ -586,22 +585,12 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
  */
 static void delete_vma_from_mm(struct vm_area_struct *vma)
 {
-       int i;
-       struct address_space *mapping;
-       struct mm_struct *mm = vma->vm_mm;
-       struct task_struct *curr = current;
        MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
 
-       mm->map_count--;
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               /* if the vma is cached, invalidate the entire cache */
-               if (curr->vmacache.vmas[i] == vma) {
-                       vmacache_invalidate(mm);
-                       break;
-               }
-       }
+       vma->vm_mm->map_count--;
        /* remove the VMA from the mapping */
        if (vma->vm_file) {
+               struct address_space *mapping;
                mapping = vma->vm_file->f_mapping;
 
                i_mmap_lock_write(mapping);
@@ -613,7 +602,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
 
        /* remove from the MM's tree and list */
        vma_mas_remove(vma, &mas);
-       __vma_unlink_list(mm, vma);
+       __vma_unlink_list(vma->vm_mm, vma);
 }
 
 /*
@@ -638,18 +627,10 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
        struct vm_area_struct *vma;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find(mm, addr);
-       if (likely(vma))
-               return vma;
-
        rcu_read_lock();
        vma = mas_walk(&mas);
        rcu_read_unlock();
 
-       if (vma)
-               vmacache_update(addr, vma);
-
        return vma;
 }
 EXPORT_SYMBOL(find_vma);
@@ -684,11 +665,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        unsigned long end = addr + len;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find_exact(mm, addr, end);
-       if (vma)
-               return vma;
-
        rcu_read_lock();
        vma = mas_walk(&mas);
        rcu_read_unlock();
@@ -699,7 +675,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        if (vma->vm_end != end)
                return NULL;
 
-       vmacache_update(addr, vma);
        return vma;
 }
 

diff --git a/mm/vmacache.c b/mm/vmacache.c
deleted file mode 100644 (file)
index 01a6e66..0000000
--- a/mm/vmacache.c
+++ /dev/null
@@ -1,117 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2014 Davidlohr Bueso.
- */
-#include <linux/sched/signal.h>
-#include <linux/sched/task.h>
-#include <linux/mm.h>
-#include <linux/vmacache.h>
-
-/*
- * Hash based on the pmd of addr if configured with MMU, which provides a good
- * hit rate for workloads with spatial locality.  Otherwise, use pages.
- */
-#ifdef CONFIG_MMU
-#define VMACACHE_SHIFT PMD_SHIFT
-#else
-#define VMACACHE_SHIFT PAGE_SHIFT
-#endif
-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
-
-/*
- * This task may be accessing a foreign mm via (for example)
- * get_user_pages()->find_vma().  The vmacache is task-local and this
- * task's vmacache pertains to a different mm (ie, its own).  There is
- * nothing we can do here.
- *
- * Also handle the case where a kernel thread has adopted this mm via
- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
- */
-static inline bool vmacache_valid_mm(struct mm_struct *mm)
-{
-       return current->mm == mm && !(current->flags & PF_KTHREAD);
-}
-
-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
-{
-       if (vmacache_valid_mm(newvma->vm_mm))
-               current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
-}
-
-static bool vmacache_valid(struct mm_struct *mm)
-{
-       struct task_struct *curr;
-
-       if (!vmacache_valid_mm(mm))
-               return false;
-
-       curr = current;
-       if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
-               /*
-                * First attempt will always be invalid, initialize
-                * the new cache for this task here.
-                */
-               curr->vmacache.seqnum = mm->vmacache_seqnum;
-               vmacache_flush(curr);
-               return false;
-       }
-       return true;
-}
-
-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
-{
-       int idx = VMACACHE_HASH(addr);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma) {
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-                       if (WARN_ON_ONCE(vma->vm_mm != mm))
-                               break;
-#endif
-                       if (vma->vm_start <= addr && vma->vm_end > addr) {
-                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                               return vma;
-                       }
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-
-#ifndef CONFIG_MMU
-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                          unsigned long start,
-                                          unsigned long end)
-{
-       int idx = VMACACHE_HASH(start);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma && vma->vm_start == start && vma->vm_end == end) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-#endif

diff --git a/mm/vmstat.c b/mm/vmstat.c
index b0534e068166c4d4df5995a2830fe548f48886b7..651ab5cc00e0c0f64a87c68cab93241768fe52ea 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1319,10 +1319,6 @@ const char * const vmstat_text[] = {
        "nr_tlb_local_flush_one",
 #endif /* CONFIG_DEBUG_TLBFLUSH */
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-       "vmacache_find_calls",
-       "vmacache_find_hits",
-#endif
 #ifdef CONFIG_SWAP
        "swap_ra",
        "swap_ra_hit",