mm: rcu safe VMA freeing

This prepares for page faults handling under VMA lock, looking up VMAs
under protection of an rcu read lock, instead of the usual mmap read lock.

Link: https://lkml.kernel.org/r/20230227173632.3292573-11-surenb@google.com
Signed-off-by: Michel Lespinasse <michel@lespinasse.org>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a57e6ae78e6569ef67058faa9e353175628de44c..7be5fe7ac495650b1018308fcea938f1bf1cc77d 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -480,9 +480,16 @@ struct anon_vma_name {
 struct vm_area_struct {
        /* The first cache line has the info for VMA tree walking. */
 
-       unsigned long vm_start;         /* Our start address within vm_mm. */
-       unsigned long vm_end;           /* The first byte after our end address
-                                          within vm_mm. */
+       union {
+               struct {
+                       /* VMA covers [vm_start; vm_end) addresses within mm */
+                       unsigned long vm_start;
+                       unsigned long vm_end;
+               };
+#ifdef CONFIG_PER_VMA_LOCK
+               struct rcu_head vm_rcu; /* Used for deferred freeing. */
+#endif
+       };
 
        struct mm_struct *vm_mm;        /* The address space we belong to. */
 

diff --git a/kernel/fork.c b/kernel/fork.c
index 32dbd7cf42ebef0b70d07ad8ad62cc20a58424e7..c0fcfe82c05e230de350c8fa71677545ffe88764 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -479,12 +479,30 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
        return new;
 }
 
-void vm_area_free(struct vm_area_struct *vma)
+static void __vm_area_free(struct vm_area_struct *vma)
 {
        free_anon_vma_name(vma);
        kmem_cache_free(vm_area_cachep, vma);
 }
 
+#ifdef CONFIG_PER_VMA_LOCK
+static void vm_area_free_rcu_cb(struct rcu_head *head)
+{
+       struct vm_area_struct *vma = container_of(head, struct vm_area_struct,
+                                                 vm_rcu);
+       __vm_area_free(vma);
+}
+#endif
+
+void vm_area_free(struct vm_area_struct *vma)
+{
+#ifdef CONFIG_PER_VMA_LOCK
+       call_rcu(&vma->vm_rcu, vm_area_free_rcu_cb);
+#else
+       __vm_area_free(vma);
+#endif
+}
+
 static void account_kernel_stack(struct task_struct *tsk, int account)
 {
        if (IS_ENABLED(CONFIG_VMAP_STACK)) {