fs/proc/task_mmu: execute PROCMAP_QUERY ioctl under per-vma locks

Utilize per-vma locks to stabilize vma after lookup without taking
mmap_lock during PROCMAP_QUERY ioctl execution. While we might take
mmap_lock for reading during contention, we do that momentarily only
to lock the vma.
This change is designed to reduce mmap_lock contention and prevent
PROCMAP_QUERY ioctl calls from blocking address space updates.

Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index e41a69f4c0bce9ea9b69a88adf1ddbb172c9b4ae..a42045e0a9aed606da9929a10fcd71bdbcdecac6 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -487,28 +487,64 @@ static int pid_maps_open(struct inode *inode, struct file *file)
                PROCMAP_QUERY_VMA_FLAGS                         \
 )
 
-static int query_vma_setup(struct mm_struct *mm)
+#ifdef CONFIG_PER_VMA_LOCK
+
+static int query_vma_setup(struct proc_maps_private *priv)
 {
-       return mmap_read_lock_killable(mm);
+       priv->locked_vma = NULL;
+       priv->mmap_locked = false;
+
+       return 0;
 }
 
-static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
+static void query_vma_teardown(struct proc_maps_private *priv)
 {
-       mmap_read_unlock(mm);
+       unlock_vma(priv);
+}
+
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
+                                                    unsigned long addr)
+{
+       struct vm_area_struct *vma;
+
+       rcu_read_lock();
+       vma_iter_init(&priv->iter, priv->mm, addr);
+       vma = get_next_vma(priv, addr);
+       rcu_read_unlock();
+
+       return vma;
 }
 
-static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr)
+#else /* CONFIG_PER_VMA_LOCK */
+
+static int query_vma_setup(struct proc_maps_private *priv)
 {
-       return find_vma(mm, addr);
+       return mmap_read_lock_killable(priv->mm);
 }
 
-static struct vm_area_struct *query_matching_vma(struct mm_struct *mm,
+static void query_vma_teardown(struct proc_maps_private *priv)
+{
+       mmap_read_unlock(priv->mm);
+}
+
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
+                                                    unsigned long addr)
+{
+       return find_vma(priv->mm, addr);
+}
+
+#endif  /* CONFIG_PER_VMA_LOCK */
+
+static struct vm_area_struct *query_matching_vma(struct proc_maps_private *priv,
                                                 unsigned long addr, u32 flags)
 {
        struct vm_area_struct *vma;
 
 next_vma:
-       vma = query_vma_find_by_addr(mm, addr);
+       vma = query_vma_find_by_addr(priv, addr);
+       if (IS_ERR(vma))
+               return vma;
+
        if (!vma)
                goto no_vma;
 
@@ -584,13 +620,13 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
        if (!mm || !mmget_not_zero(mm))
                return -ESRCH;
 
-       err = query_vma_setup(mm);
+       err = query_vma_setup(priv);
        if (err) {
                mmput(mm);
                return err;
        }
 
-       vma = query_matching_vma(mm, karg.query_addr, karg.query_flags);
+       vma = query_matching_vma(priv, karg.query_addr, karg.query_flags);
        if (IS_ERR(vma)) {
                err = PTR_ERR(vma);
                vma = NULL;
@@ -675,7 +711,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
        }
 
        /* unlock vma or mmap_lock, and put mm_struct before copying data to user */
-       query_vma_teardown(mm, vma);
+       query_vma_teardown(priv);
        mmput(mm);
 
        if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr),
@@ -695,7 +731,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
        return 0;
 
 out:
-       query_vma_teardown(mm, vma);
+       query_vma_teardown(priv);
        mmput(mm);
        kfree(name_buf);
        return err;