bpf: Fix usage of trace RCU in local storage.

bpf_{sk,task,inode}_storage_free() do not need to use
call_rcu_tasks_trace as no BPF program should be accessing the owner
as it's being destroyed. The only other reader at this point is
bpf_local_storage_map_free() which uses normal RCU.

The only path that needs trace RCU are:

* bpf_local_storage_{delete,update} helpers
* map_{delete,update}_elem() syscalls

Fixes: 0fe4b381a59e ("bpf: Allow bpf_local_storage to be used by sleepable programs")
Signed-off-by: KP Singh <kpsingh@kernel.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Link: https://lore.kernel.org/bpf/20220418155158.2865678-1-kpsingh@kernel.org

diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
index 493e6325849706481c480fcff5259f559da049af..7ea18d4da84b8840264d1ef21de1fef86e415bb5 100644 (file)
--- a/include/linux/bpf_local_storage.h
+++ b/include/linux/bpf_local_storage.h
@@ -143,9 +143,9 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 
 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
                                     struct bpf_local_storage_elem *selem,
-                                    bool uncharge_omem);
+                                    bool uncharge_omem, bool use_trace_rcu);
 
-void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
+void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
 
 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
                        struct bpf_local_storage_elem *selem);

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index 96be8d518885c8fc024ad890e36ead2c9e49835a..10424a1cda51d4219d98637e6adbccd19c447d71 100644 (file)
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -90,7 +90,7 @@ void bpf_inode_storage_free(struct inode *inode)
                 */
                bpf_selem_unlink_map(selem);
                free_inode_storage = bpf_selem_unlink_storage_nolock(
-                       local_storage, selem, false);
+                       local_storage, selem, false, false);
        }
        raw_spin_unlock_bh(&local_storage->lock);
        rcu_read_unlock();
@@ -149,7 +149,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata));
+       bpf_selem_unlink(SELEM(sdata), true);
 
        return 0;
 }

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 01aa2b51ec4dfe3f38d7653050a275640c14920c..8ce40fd869f6a7c7099c4bc9892fea7b4b382f3b 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -106,7 +106,7 @@ static void bpf_selem_free_rcu(struct rcu_head *rcu)
  */
 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
                                     struct bpf_local_storage_elem *selem,
-                                    bool uncharge_mem)
+                                    bool uncharge_mem, bool use_trace_rcu)
 {
        struct bpf_local_storage_map *smap;
        bool free_local_storage;
@@ -150,11 +150,16 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
            SDATA(selem))
                RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
 
-       call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);
+       if (use_trace_rcu)
+               call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);
+       else
+               kfree_rcu(selem, rcu);
+
        return free_local_storage;
 }
 
-static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)
+static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
+                                      bool use_trace_rcu)
 {
        struct bpf_local_storage *local_storage;
        bool free_local_storage = false;
@@ -169,12 +174,16 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)
        raw_spin_lock_irqsave(&local_storage->lock, flags);
        if (likely(selem_linked_to_storage(selem)))
                free_local_storage = bpf_selem_unlink_storage_nolock(
-                       local_storage, selem, true);
+                       local_storage, selem, true, use_trace_rcu);
        raw_spin_unlock_irqrestore(&local_storage->lock, flags);
 
-       if (free_local_storage)
-               call_rcu_tasks_trace(&local_storage->rcu,
+       if (free_local_storage) {
+               if (use_trace_rcu)
+                       call_rcu_tasks_trace(&local_storage->rcu,
                                     bpf_local_storage_free_rcu);
+               else
+                       kfree_rcu(local_storage, rcu);
+       }
 }
 
 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
@@ -214,14 +223,14 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap,
        raw_spin_unlock_irqrestore(&b->lock, flags);
 }
 
-void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
+void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu)
 {
        /* Always unlink from map before unlinking from local_storage
         * because selem will be freed after successfully unlinked from
         * the local_storage.
         */
        bpf_selem_unlink_map(selem);
-       __bpf_selem_unlink_storage(selem);
+       __bpf_selem_unlink_storage(selem, use_trace_rcu);
 }
 
 struct bpf_local_storage_data *
@@ -466,7 +475,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
        if (old_sdata) {
                bpf_selem_unlink_map(SELEM(old_sdata));
                bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
-                                               false);
+                                               false, true);
        }
 
 unlock:
@@ -548,7 +557,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
                                migrate_disable();
                                __this_cpu_inc(*busy_counter);
                        }
-                       bpf_selem_unlink(selem);
+                       bpf_selem_unlink(selem, false);
                        if (busy_counter) {
                                __this_cpu_dec(*busy_counter);
                                migrate_enable();

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index 6638a0ecc3d21531f1c8628c7df62366ca682101..57904263a710f628e12952f89421c4a73b8c77c8 100644 (file)
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -102,7 +102,7 @@ void bpf_task_storage_free(struct task_struct *task)
                 */
                bpf_selem_unlink_map(selem);
                free_task_storage = bpf_selem_unlink_storage_nolock(
-                       local_storage, selem, false);
+                       local_storage, selem, false, false);
        }
        raw_spin_unlock_irqrestore(&local_storage->lock, flags);
        bpf_task_storage_unlock();
@@ -192,7 +192,7 @@ static int task_storage_delete(struct task_struct *task, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata));
+       bpf_selem_unlink(SELEM(sdata), true);
 
        return 0;
 }

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index e3ac3638052031b1fe32120b263382facc58b3bf..83d7641ef67b03bf0eed3d7acc1d366b835f3e08 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -40,7 +40,7 @@ static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata));
+       bpf_selem_unlink(SELEM(sdata), true);
 
        return 0;
 }
@@ -75,8 +75,8 @@ void bpf_sk_storage_free(struct sock *sk)
                 * sk_storage.
                 */
                bpf_selem_unlink_map(selem);
-               free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage,
-                                                                 selem, true);
+               free_sk_storage = bpf_selem_unlink_storage_nolock(
+                       sk_storage, selem, true, false);
        }
        raw_spin_unlock_bh(&sk_storage->lock);
        rcu_read_unlock();