bcachefs: rcu_pending now works in userspace

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/rcu_pending.c b/fs/bcachefs/rcu_pending.c
index 8f8d914d399885d856fd7d354c479561f402a78e..40a20192eee8915af49bce0873caf9af7ea5e830 100644 (file)
--- a/fs/bcachefs/rcu_pending.c
+++ b/fs/bcachefs/rcu_pending.c
@@ -115,24 +115,40 @@ static bool __rcu_pending_has_pending(struct rcu_pending_pcpu *p)
 static void rcu_pending_list_merge(struct rcu_pending_list *l1,
                                   struct rcu_pending_list *l2)
 {
+#ifdef __KERNEL__
        if (!l1->head)
                l1->head = l2->head;
        else
                l1->tail->next = l2->head;
-       l1->tail = l2->tail;
+#else
+       if (!l1->head)
+               l1->head = l2->head;
+       else
+               l1->tail->next.next = (void *) l2->head;
+#endif
 
+       l1->tail = l2->tail;
        l2->head = l2->tail = NULL;
 }
 
 static void rcu_pending_list_add(struct rcu_pending_list *l,
                                 struct rcu_head *n)
 {
+#ifdef __KERNEL__
        if (!l->head)
                l->head = n;
        else
                l->tail->next = n;
        l->tail = n;
        n->next = NULL;
+#else
+       if (!l->head)
+               l->head = n;
+       else
+               l->tail->next.next = (void *) n;
+       l->tail = n;
+       n->next.next = NULL;
+#endif
 }
 
 static void merge_expired_lists(struct rcu_pending_pcpu *p)
@@ -144,6 +160,19 @@ static void merge_expired_lists(struct rcu_pending_pcpu *p)
                        rcu_pending_list_merge(expired, i);
 }
 
+#ifndef __KERNEL__
+static inline void kfree_bulk(size_t nr, void ** p)
+{
+       while (nr--)
+               kfree(*p);
+}
+
+#define local_irq_save(flags)          \
+do {                                   \
+       flags = 0;                      \
+} while (0)
+#endif
+
 static noinline void __process_finished_items(struct rcu_pending *pending,
                                              struct rcu_pending_pcpu *p,
                                              unsigned long flags)
@@ -177,7 +206,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
 
                while (list) {
                        struct rcu_head *obj = list;
+#ifdef __KERNEL__
                        list = obj->next;
+#else
+                       list = (void *) obj->next.next;
+#endif
 
                        /*
                         * low bit of pointer indicates whether rcu_head needs
@@ -204,7 +237,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
 
                while (list) {
                        struct rcu_head *obj = list;
+#ifdef __KERNEL__
                        list = obj->next;
+#else
+                       list = (void *) obj->next.next;
+#endif
                        obj->func(obj);
                }
                break;
@@ -216,7 +253,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending,
 
                while (list) {
                        struct rcu_head *obj = list;
+#ifdef __KERNEL__
                        list = obj->next;
+#else
+                       list = (void *) obj->next.next;
+#endif
                        pending->process(pending, obj);
                }
                break;
@@ -265,11 +306,13 @@ static void rcu_pending_rcu_cb(struct rcu_head *rcu)
 
        unsigned long flags;
        spin_lock_irqsave(&p->lock, flags);
-       if (__rcu_pending_has_pending(p))
+       if (__rcu_pending_has_pending(p)) {
+               spin_unlock_irqrestore(&p->lock, flags);
                __call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb);
-       else
+       } else {
                p->cb_armed = false;
-       spin_unlock_irqrestore(&p->lock, flags);
+               spin_unlock_irqrestore(&p->lock, flags);
+       }
 }
 
 static __always_inline struct rcu_pending_seq *
@@ -489,7 +532,11 @@ static struct rcu_head *rcu_pending_pcpu_dequeue(struct rcu_pending_pcpu *p)
        static_array_for_each(p->lists, i)
                if (i->head) {
                        ret = i->head;
+#ifdef __KERNEL__
                        i->head = ret->next;
+#else
+                       i->head = (void *) ret->next.next;
+#endif
                        if (!i->head)
                                i->tail = NULL;
                        goto out;

diff --git a/fs/bcachefs/rcu_pending.h b/fs/bcachefs/rcu_pending.h
index a875c640da8de40162f2eac2b2beaaae403cb68a..71a2f4ddaade484ca2e24992a8ac9f5352abd633 100644 (file)
--- a/fs/bcachefs/rcu_pending.h
+++ b/fs/bcachefs/rcu_pending.h
@@ -2,6 +2,8 @@
 #ifndef _LINUX_RCU_PENDING_H
 #define _LINUX_RCU_PENDING_H
 
+#include <linux/rcupdate.h>
+
 struct rcu_pending;
 typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *);