mm: introduce lock_vma_under_rcu to be used from arch-specific code

Introduce lock_vma_under_rcu function to lookup and lock a VMA during
page fault handling. When VMA is not found, can't be locked or changes
after being locked, the function returns NULL. The lookup is performed
under RCU protection to prevent the found VMA from being destroyed before
the VMA lock is acquired. VMA lock statistics are updated according to
the results.
For now only anonymous VMAs can be searched this way. In other cases the
function returns NULL.

Signed-off-by: Suren Baghdasaryan <surenb@google.com>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 3851f786c95aff13094cda771acf99b25d47f53a..d2c1e03c8088ad1e6d6ce19e449feb7628a875fc 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -684,6 +684,9 @@ static inline void vma_assert_no_reader(struct vm_area_struct *vma)
                      vma);
 }
 
+struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
+                                         unsigned long address);
+
 #else /* CONFIG_PER_VMA_LOCK */
 
 static inline void vma_init_lock(struct vm_area_struct *vma) {}

diff --git a/mm/memory.c b/mm/memory.c
index a5a33d88f3c0379d6c9728ac04ef7abfa7a845b9..89b14a05f24896ce324d97873af8e61ddc1e6919 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5241,6 +5241,51 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 }
 EXPORT_SYMBOL_GPL(handle_mm_fault);
 
+#ifdef CONFIG_PER_VMA_LOCK
+/*
+ * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
+ * stable and not isolated. If the VMA is not found or is being modified the
+ * function returns NULL.
+ */
+struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
+                                         unsigned long address)
+{
+       MA_STATE(mas, &mm->mm_mt, address, address);
+       struct vm_area_struct *vma, *validate;
+
+       rcu_read_lock();
+       vma = mas_walk(&mas);
+retry:
+       if (!vma)
+               goto inval;
+
+       if (!vma_is_anonymous(vma))
+               goto inval;
+
+       if (!vma_read_trylock(vma)) {
+               count_vm_vma_lock_event(VMA_LOCK_ABORT);
+               goto inval;
+       }
+
+       /* Check if the VMA got isolated after we found it */
+       mas.index = address;
+       validate = mas_walk(&mas);
+       if (validate != vma) {
+               vma_read_unlock(vma);
+               count_vm_vma_lock_event(VMA_LOCK_MISS);
+               /* The area was replaced with another one. */
+               vma = validate;
+               goto retry;
+       }
+
+       rcu_read_unlock();
+       return vma;
+inval:
+       rcu_read_unlock();
+       return NULL;
+}
+#endif /* CONFIG_PER_VMA_LOCK */
+
 #ifndef __PAGETABLE_P4D_FOLDED
 /*
  * Allocate p4d page table.