mm: remove vmacache

By using the maple tree and the maple tree state, the vmacache is no
longer beneficial and is complicating the VMA code.  Remove the vmacache
to reduce the work in keeping it up to date and code complexity.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/fs/exec.c b/fs/exec.c
index 14b4b3755580ca85d019848b3cee9f34f13c862c..b856224b853546a790c1eff2eb0e87cdd618b7e6 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -28,7 +28,6 @@
 #include <linux/file.h>
 #include <linux/fdtable.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/stat.h>
 #include <linux/fcntl.h>
 #include <linux/swap.h>
@@ -1027,8 +1026,6 @@ static int exec_mmap(struct mm_struct *mm)
        activate_mm(active_mm, mm);
        if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
                local_irq_enable();
-       tsk->mm->vmacache_seqnum = 0;
-       vmacache_flush(tsk);
        task_unlock(tsk);
        if (old_mm) {
                mmap_read_unlock(old_mm);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index d7de4584a27141677bb7a7e3fae945dd1b90f190..17e87eab7a9e384461146186a5a426a38bddb7b0 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/pagewalk.h>
-#include <linux/vmacache.h>
 #include <linux/mm_inline.h>
 #include <linux/hugetlb.h>
 #include <linux/huge_mm.h>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 882283ad0cfb62e2cc2d2f0cfd7f91da1bfa2666..ca47e66d1b147b7022aa73ef8da8ddececd5c717 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -459,7 +459,6 @@ struct mm_struct {
        struct {
                struct vm_area_struct *mmap;            /* list of VMAs */
                struct maple_tree mm_mt;
-               u64 vmacache_seqnum;                   /* per-thread vmacache */
 #ifdef CONFIG_MMU
                unsigned long (*get_unmapped_area) (struct file *filp,
                                unsigned long addr, unsigned long len,

diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index c1bc6731125cbbeb99b2bbe0d4757fad6bf5031f..0bb4b6da9993942e320c424990ea8397149b37c8 100644 (file)
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -24,18 +24,6 @@
                IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
 #define ALLOC_SPLIT_PTLOCKS    (SPINLOCK_SIZE > BITS_PER_LONG/8)
 
-/*
- * The per task VMA cache array:
- */
-#define VMACACHE_BITS 2
-#define VMACACHE_SIZE (1U << VMACACHE_BITS)
-#define VMACACHE_MASK (VMACACHE_SIZE - 1)
-
-struct vmacache {
-       u64 seqnum;
-       struct vm_area_struct *vmas[VMACACHE_SIZE];
-};
-
 /*
  * When updating this, please also update struct resident_page_types[] in
  * kernel/fork.c

diff --git a/include/linux/sched.h b/include/linux/sched.h
index d5e3c00b74e12546c44faad9e00f85a8501ccf98..e7f984a7311e6065299f5cbffdd2f26722890662 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -861,7 +861,6 @@ struct task_struct {
        struct mm_struct                *active_mm;
 
        /* Per-thread vma caching: */
-       struct vmacache                 vmacache;
 
 #ifdef SPLIT_RSS_COUNTING
        struct task_rss_stat            rss_stat;

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 16a0a4fd000be0a62eaf5ac33cb66c95808dd59b..aef0369e9c08447f88cbc11dfb3273cb452f711a 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -122,10 +122,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                NR_TLB_LOCAL_FLUSH_ALL,
                NR_TLB_LOCAL_FLUSH_ONE,
 #endif /* CONFIG_DEBUG_TLBFLUSH */
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-               VMACACHE_FIND_CALLS,
-               VMACACHE_FIND_HITS,
-#endif
 #ifdef CONFIG_SWAP
                SWAP_RA,
                SWAP_RA_HIT,

diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h
deleted file mode 100644 (file)
index 6fce268..0000000
--- a/include/linux/vmacache.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_VMACACHE_H
-#define __LINUX_VMACACHE_H
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-
-static inline void vmacache_flush(struct task_struct *tsk)
-{
-       memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
-}
-
-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma);
-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm,
-                                                   unsigned long addr);
-
-#ifndef CONFIG_MMU
-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                                 unsigned long start,
-                                                 unsigned long end);
-#endif
-
-static inline void vmacache_invalidate(struct mm_struct *mm)
-{
-       mm->vmacache_seqnum++;
-}
-
-#endif /* __LINUX_VMACACHE_H */

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index bfe38869498d7cfa199cfc653d3b4b63f854b371..19cf5b6892ceba12c9f8acfa0a7910b4a2e726d4 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu)
 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
 #endif
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-#define count_vm_vmacache_event(x) count_vm_event(x)
-#else
-#define count_vm_vmacache_event(x) do {} while (0)
-#endif
-
 #define __count_zid_vm_events(item, zid, delta) \
        __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
 

diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index da06a5553835b5e443555ee6b446e93abc2c3ffe..c4e6f5159bedb64878af7562f21eaef4e55b8153 100644 (file)
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -50,7 +50,6 @@
 #include <linux/pid.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/rcupdate.h>
 #include <linux/irq.h>
 
@@ -282,17 +281,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)
        if (!CACHE_FLUSH_IS_SAFE)
                return;
 
-       if (current->mm) {
-               int i;
-
-               for (i = 0; i < VMACACHE_SIZE; i++) {
-                       if (!current->vmacache.vmas[i])
-                               continue;
-                       flush_cache_range(current->vmacache.vmas[i],
-                                         addr, addr + BREAK_INSTR_SIZE);
-               }
-       }
-
        /* Force flush instruction cache if it was outside the mm */
        flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
 }

diff --git a/kernel/fork.c b/kernel/fork.c
index cb5a1423ea27aee1419f33ebd051a8acc82a6ee1..20890be1371bc083feb7df16be2fbd0dbf3c97bb 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -43,7 +43,6 @@
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/mm_inline.h>
-#include <linux/vmacache.h>
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
@@ -1116,7 +1115,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
        mm->mmap = NULL;
        mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
        mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
-       mm->vmacache_seqnum = 0;
        atomic_set(&mm->mm_users, 1);
        atomic_set(&mm->mm_count, 1);
        seqcount_init(&mm->write_protect_seq);
@@ -1572,9 +1570,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
        if (!oldmm)
                return 0;
 
-       /* initialize the new vmacache entries */
-       vmacache_flush(tsk);
-
        if (clone_flags & CLONE_VM) {
                mmget(oldmm);
                mm = oldmm;

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 3d9366075153db9d159b1080b21affa741dc7aa1..8a0567046e9eefbc4c48d9c26e6d1406666e7e3f 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -837,14 +837,6 @@ config DEBUG_VM
 
          If unsure, say N.
 
-config DEBUG_VM_VMACACHE
-       bool "Debug VMA caching"
-       depends on DEBUG_VM
-       help
-         Enable this to turn on VMA caching debug information. Doing so
-         can cause significant overhead, so only enable it in non-production
-         environments.
-
 config DEBUG_VM_MAPLE_TREE
        bool "Debug VM maple trees"
        depends on DEBUG_VM

diff --git a/mm/Makefile b/mm/Makefile
index 4cc13f3179a518064ae121d531b9a9ddfec734c2..f716e8a570073e54b16aeafc2d618014e9edba4a 100644 (file)
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -52,7 +52,7 @@ obj-y                 := filemap.o mempool.o oom_kill.o fadvise.o \
                           readahead.o swap.o truncate.o vmscan.o shmem.o \
                           util.o mmzone.o vmstat.o backing-dev.o \
                           mm_init.o percpu.o slab_common.o \
-                          compaction.o vmacache.o \
+                          compaction.o \
                           interval_tree.o list_lru.o workingset.o \
                           debug.o gup.o mmap_lock.o $(mmu-y)
 

diff --git a/mm/debug.c b/mm/debug.c
index bef329bf28f01aef04256db49f0790fcb65ea8d1..2d625ca0e3269408e7e4e0ce325d174658ce6644 100644 (file)
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -155,7 +155,7 @@ EXPORT_SYMBOL(dump_vma);
 
 void dump_mm(const struct mm_struct *mm)
 {
-       pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n"
+       pr_emerg("mm %px mmap %px task_size %lu\n"
 #ifdef CONFIG_MMU
                "get_unmapped_area %px\n"
 #endif
@@ -183,7 +183,7 @@ void dump_mm(const struct mm_struct *mm)
                "tlb_flush_pending %d\n"
                "def_flags: %#lx(%pGv)\n",
 
-               mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size,
+               mm, mm->mmap, mm->task_size,
 #ifdef CONFIG_MMU
                mm->get_unmapped_area,
 #endif

diff --git a/mm/mmap.c b/mm/mmap.c
index 2d7d9c26c57cc7d8d7c98e63b51acc87155aa5b5..494080c9bcc7ac448a14f4fbf6e1b040e63fdb33 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -14,7 +14,6 @@
 #include <linux/backing-dev.h>
 #include <linux/mm.h>
 #include <linux/mm_inline.h>
-#include <linux/vmacache.h>
 #include <linux/shm.h>
 #include <linux/mman.h>
 #include <linux/pagemap.h>
@@ -707,9 +706,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
                /* Remove from mm linked list - also updates highest_vm_end */
                __vma_unlink_list(mm, next);
 
-               /* Kill the cache */
-               vmacache_invalidate(mm);
-
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
 
@@ -948,8 +944,6 @@ again:
 
        if (remove_next) {
                __vma_unlink_list(mm, next);
-               /* Kill the cache */
-               vmacache_invalidate(mm);
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
        } else if (insert) {
@@ -2229,19 +2223,10 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
                                             unsigned long start_addr,
                                             unsigned long end_addr)
 {
-       struct vm_area_struct *vma;
        unsigned long index = start_addr;
 
        mmap_assert_locked(mm);
-       /* Check the cache first. */
-       vma = vmacache_find(mm, start_addr);
-       if (likely(vma))
-               return vma;
-
-       vma = mt_find(&mm->mm_mt, &index, end_addr - 1);
-       if (vma)
-               vmacache_update(start_addr, vma);
-       return vma;
+       return mt_find(&mm->mm_mt, &index, end_addr - 1);
 }
 EXPORT_SYMBOL(find_vma_intersection);
 
@@ -2255,19 +2240,10 @@ EXPORT_SYMBOL(find_vma_intersection);
  */
 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
 {
-       struct vm_area_struct *vma;
        unsigned long index = addr;
 
        mmap_assert_locked(mm);
-       /* Check the cache first. */
-       vma = vmacache_find(mm, addr);
-       if (likely(vma))
-               return vma;
-
-       vma = mt_find(&mm->mm_mt, &index, ULONG_MAX);
-       if (vma)
-               vmacache_update(addr, vma);
-       return vma;
+       return mt_find(&mm->mm_mt, &index, ULONG_MAX);
 }
 EXPORT_SYMBOL(find_vma);
 
@@ -2642,9 +2618,6 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
        tail_vma->vm_next = NULL;
 
-       /* Kill the cache */
-       vmacache_invalidate(mm);
-
        /*
         * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
         * VM_GROWSUP VMA. Such VMAs can change their size under
@@ -3011,7 +2984,6 @@ static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
 
        vma_mas_remove(&unmap, mas);
 
-       vmacache_invalidate(vma->vm_mm);
        if (vma->anon_vma) {
                anon_vma_interval_tree_post_update_vma(vma);
                anon_vma_unlock_write(vma->anon_vma);

diff --git a/mm/nommu.c b/mm/nommu.c
index 7d8f756f13f36ee6440c3fd1e17a2d76e13abbc3..26a9056e508f0840bae4155222a8bf81e98f3c44 100644 (file)
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -19,7 +19,6 @@
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/sched/mm.h>
-#include <linux/vmacache.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/file.h>
@@ -583,23 +582,12 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
  */
 static void delete_vma_from_mm(struct vm_area_struct *vma)
 {
-       int i;
-       struct address_space *mapping;
-       struct mm_struct *mm = vma->vm_mm;
-       struct task_struct *curr = current;
        MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
 
-       mm->map_count--;
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               /* if the vma is cached, invalidate the entire cache */
-               if (curr->vmacache.vmas[i] == vma) {
-                       vmacache_invalidate(mm);
-                       break;
-               }
-       }
-
+       vma->vm_mm->map_count--;
        /* remove the VMA from the mapping */
        if (vma->vm_file) {
+               struct address_space *mapping;
                mapping = vma->vm_file->f_mapping;
 
                i_mmap_lock_write(mapping);
@@ -611,7 +599,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
 
        /* remove from the MM's tree and list */
        vma_mas_remove(vma, &mas);
-       __vma_unlink_list(mm, vma);
+       __vma_unlink_list(vma->vm_mm, vma);
 }
 
 /*
@@ -644,20 +632,9 @@ EXPORT_SYMBOL(find_vma_intersection);
  */
 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
 {
-       struct vm_area_struct *vma;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find(mm, addr);
-       if (likely(vma))
-               return vma;
-
-       vma = mas_walk(&mas);
-
-       if (vma)
-               vmacache_update(addr, vma);
-
-       return vma;
+       return mas_walk(&mas);
 }
 EXPORT_SYMBOL(find_vma);
 
@@ -691,11 +668,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        unsigned long end = addr + len;
        MA_STATE(mas, &mm->mm_mt, addr, addr);
 
-       /* check the cache first */
-       vma = vmacache_find_exact(mm, addr, end);
-       if (vma)
-               return vma;
-
        vma = mas_walk(&mas);
        if (!vma)
                return NULL;
@@ -704,7 +676,6 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
        if (vma->vm_end != end)
                return NULL;
 
-       vmacache_update(addr, vma);
        return vma;
 }
 

diff --git a/mm/vmacache.c b/mm/vmacache.c
deleted file mode 100644 (file)
index 01a6e66..0000000
--- a/mm/vmacache.c
+++ /dev/null
@@ -1,117 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2014 Davidlohr Bueso.
- */
-#include <linux/sched/signal.h>
-#include <linux/sched/task.h>
-#include <linux/mm.h>
-#include <linux/vmacache.h>
-
-/*
- * Hash based on the pmd of addr if configured with MMU, which provides a good
- * hit rate for workloads with spatial locality.  Otherwise, use pages.
- */
-#ifdef CONFIG_MMU
-#define VMACACHE_SHIFT PMD_SHIFT
-#else
-#define VMACACHE_SHIFT PAGE_SHIFT
-#endif
-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
-
-/*
- * This task may be accessing a foreign mm via (for example)
- * get_user_pages()->find_vma().  The vmacache is task-local and this
- * task's vmacache pertains to a different mm (ie, its own).  There is
- * nothing we can do here.
- *
- * Also handle the case where a kernel thread has adopted this mm via
- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
- */
-static inline bool vmacache_valid_mm(struct mm_struct *mm)
-{
-       return current->mm == mm && !(current->flags & PF_KTHREAD);
-}
-
-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
-{
-       if (vmacache_valid_mm(newvma->vm_mm))
-               current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
-}
-
-static bool vmacache_valid(struct mm_struct *mm)
-{
-       struct task_struct *curr;
-
-       if (!vmacache_valid_mm(mm))
-               return false;
-
-       curr = current;
-       if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
-               /*
-                * First attempt will always be invalid, initialize
-                * the new cache for this task here.
-                */
-               curr->vmacache.seqnum = mm->vmacache_seqnum;
-               vmacache_flush(curr);
-               return false;
-       }
-       return true;
-}
-
-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
-{
-       int idx = VMACACHE_HASH(addr);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma) {
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-                       if (WARN_ON_ONCE(vma->vm_mm != mm))
-                               break;
-#endif
-                       if (vma->vm_start <= addr && vma->vm_end > addr) {
-                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                               return vma;
-                       }
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-
-#ifndef CONFIG_MMU
-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
-                                          unsigned long start,
-                                          unsigned long end)
-{
-       int idx = VMACACHE_HASH(start);
-       int i;
-
-       count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
-       if (!vmacache_valid(mm))
-               return NULL;
-
-       for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
-               if (vma && vma->vm_start == start && vma->vm_end == end) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
-               }
-               if (++idx == VMACACHE_SIZE)
-                       idx = 0;
-       }
-
-       return NULL;
-}
-#endif

diff --git a/mm/vmstat.c b/mm/vmstat.c
index b75b1a64b54cb4cc9bcd523403214795c9ee7e2d..1fd82a0c6b51d0935fc2e994c0ceb2b82fa473e5 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1382,10 +1382,6 @@ const char * const vmstat_text[] = {
        "nr_tlb_local_flush_one",
 #endif /* CONFIG_DEBUG_TLBFLUSH */
 
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-       "vmacache_find_calls",
-       "vmacache_find_hits",
-#endif
 #ifdef CONFIG_SWAP
        "swap_ra",
        "swap_ra_hit",