mm, vmacache: hash addresses based on pmd

When perf profiling a wide variety of different workloads, it was found
that vmacache_find() had higher than expected cost: up to 0.08% of cpu
utilization in some cases.  This was found to rival other core VM
functions such as alloc_pages_vma() with thp enabled and default
mempolicy, and the conditionals in __get_vma_policy().

VMACACHE_HASH() determines which of the four per-task_struct slots a vma
is cached for a particular address.  This currently depends on the pfn,
so pfn 5212 occupies a different vmacache slot than its neighboring pfn
5213.

vmacache_find() iterates through all four of current's vmacache slots
when looking up an address.  Hashing based on pfn, an address has
~1/VMACACHE_SIZE chance of being cached in the first vmacache slot, or
about 25%, *if* the vma is cached.

This patch hashes an address by its pmd instead of pte to optimize for
workloads with good spatial locality.  This results in a higher
probability of vmas being cached in the first slot that is checked:
normally ~70% on the same workloads instead of 25%.

[rientjes@google.com: various updates]
Link: http://lkml.kernel.org/r/alpine.DEB.2.21.1807231532290.109445@chino.kir.corp.google.com
Link: http://lkml.kernel.org/r/alpine.DEB.2.21.1807091749150.114630@chino.kir.corp.google.com
Signed-off-by: David Rientjes <rientjes@google.com>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h
index a5b3aa8d281f869ae21a5edd9137e0ee25ef4262..3e9a963edd6a89f2d8d34616690dba2888aed7f7 100644 (file)
--- a/include/linux/vmacache.h
+++ b/include/linux/vmacache.h
@@ -5,12 +5,6 @@
 #include <linux/sched.h>
 #include <linux/mm.h>
 
-/*
- * Hash based on the page number. Provides a good hit rate for
- * workloads with good locality and those with random accesses as well.
- */
-#define VMACACHE_HASH(addr) ((addr >> PAGE_SHIFT) & VMACACHE_MASK)
-
 static inline void vmacache_flush(struct task_struct *tsk)
 {
        memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));

diff --git a/mm/vmacache.c b/mm/vmacache.c
index db7596eb6132e8118b73e9c851563cde4a6d376a..ea517bef7dc552a10ac6ed424ca06aa50670c303 100644 (file)
--- a/mm/vmacache.c
+++ b/mm/vmacache.c
@@ -6,6 +6,18 @@
 #include <linux/sched/task.h>
 #include <linux/mm.h>
 #include <linux/vmacache.h>
+#include <asm/pgtable.h>
+
+/*
+ * Hash based on the pmd of addr if configured with MMU, which provides a good
+ * hit rate for workloads with spatial locality.  Otherwise, use pages.
+ */
+#ifdef CONFIG_MMU
+#define VMACACHE_SHIFT PMD_SHIFT
+#else
+#define VMACACHE_SHIFT PAGE_SHIFT
+#endif
+#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
 
 /*
  * Flush vma caches for threads that share a given mm.
@@ -87,6 +99,7 @@ static bool vmacache_valid(struct mm_struct *mm)
 
 struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
 {
+       int idx = VMACACHE_HASH(addr);
        int i;
 
        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -95,16 +108,20 @@ struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
                return NULL;
 
        for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[i];
+               struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
-               if (!vma)
-                       continue;
-               if (WARN_ON_ONCE(vma->vm_mm != mm))
-                       break;
-               if (vma->vm_start <= addr && vma->vm_end > addr) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
+               if (vma) {
+#ifdef CONFIG_DEBUG_VM_VMACACHE
+                       if (WARN_ON_ONCE(vma->vm_mm != mm))
+                               break;
+#endif
+                       if (vma->vm_start <= addr && vma->vm_end > addr) {
+                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
+                               return vma;
+                       }
                }
+               if (++idx == VMACACHE_SIZE)
+                       idx = 0;
        }
 
        return NULL;
@@ -115,6 +132,7 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                                           unsigned long start,
                                           unsigned long end)
 {
+       int idx = VMACACHE_HASH(start);
        int i;
 
        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -123,12 +141,14 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                return NULL;
 
        for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[i];
+               struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
                if (vma && vma->vm_start == start && vma->vm_end == end) {
                        count_vm_vmacache_event(VMACACHE_FIND_HITS);
                        return vma;
                }
+               if (++idx == VMACACHE_SIZE)
+                       idx = 0;
        }
 
        return NULL;