netfilter: nft_set_pipapo: Use nested-BH locking for nft_pipapo_scratch

nft_pipapo_scratch is a per-CPU variable and relies on disabled BH for
its locking. Without per-CPU locking in local_bh_disable() on PREEMPT_RT
this data structure requires explicit locking.

Add a local_lock_t to the data structure and use local_lock_nested_bh() for
locking. This change adds only lockdep coverage and does not alter the
functional behaviour for !PREEMPT_RT.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Florian Westphal <fw@strlen.de>

diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c
index 96b7539f5506acce27da4cd8503008592a808912..b385cfcf886f9812bc0cdde09cc031be124ef4dc 100644 (file)
--- a/net/netfilter/nft_set_pipapo.c
+++ b/net/netfilter/nft_set_pipapo.c
@@ -429,6 +429,7 @@ static struct nft_pipapo_elem *pipapo_get_slow(const struct nft_pipapo_match *m,
        scratch = *raw_cpu_ptr(m->scratch);
        if (unlikely(!scratch))
                goto out;
+       __local_lock_nested_bh(&scratch->bh_lock);
 
        map_index = scratch->map_index;
 
@@ -465,6 +466,7 @@ next_match:
                                  last);
                if (b < 0) {
                        scratch->map_index = map_index;
+                       __local_unlock_nested_bh(&scratch->bh_lock);
                        local_bh_enable();
 
                        return NULL;
@@ -484,6 +486,7 @@ next_match:
                         * *next* bitmap (not initial) for the next packet.
                         */
                        scratch->map_index = map_index;
+                       __local_unlock_nested_bh(&scratch->bh_lock);
                        local_bh_enable();
                        return e;
                }
@@ -498,6 +501,7 @@ next_match:
                data += NFT_PIPAPO_GROUPS_PADDING(f);
        }
 
+       __local_unlock_nested_bh(&scratch->bh_lock);
 out:
        local_bh_enable();
        return NULL;
@@ -1215,6 +1219,7 @@ static int pipapo_realloc_scratch(struct nft_pipapo_match *clone,
                }
 
                pipapo_free_scratch(clone, i);
+               local_lock_init(&scratch->bh_lock);
                *per_cpu_ptr(clone->scratch, i) = scratch;
        }
 

diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h
index e10cdbaa65d8864a2c9174780a82cb44776709af..eaab422aa56ab21b8531eaae81e988c49cee2f32 100644 (file)
--- a/net/netfilter/nft_set_pipapo.h
+++ b/net/netfilter/nft_set_pipapo.h
@@ -124,10 +124,12 @@ struct nft_pipapo_field {
 
 /**
  * struct nft_pipapo_scratch - percpu data used for lookup and matching
+ * @bh_lock:    PREEMPT_RT local spinlock
  * @map_index: Current working bitmap index, toggled between field matches
  * @__map:     store partial matching results during lookup
  */
 struct nft_pipapo_scratch {
+       local_lock_t bh_lock;
        u8 map_index;
        unsigned long __map[];
 };

diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c
index f0d8c796d7318695be4ab8ba21d159fb3a81a9a0..29326f3fcaf3f746101df330b6a07ebb0368f3c5 100644 (file)
--- a/net/netfilter/nft_set_pipapo_avx2.c
+++ b/net/netfilter/nft_set_pipapo_avx2.c
@@ -1163,6 +1163,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m,
        if (unlikely(!scratch))
                return NULL;
 
+       __local_lock_nested_bh(&scratch->bh_lock);
        map_index = scratch->map_index;
        map = NFT_PIPAPO_LT_ALIGN(&scratch->__map[0]);
        res  = map + (map_index ? m->bsize_max : 0);
@@ -1228,6 +1229,7 @@ next_match:
                if (ret < 0) {
                        scratch->map_index = map_index;
                        kernel_fpu_end();
+                       __local_unlock_nested_bh(&scratch->bh_lock);
                        return NULL;
                }
 
@@ -1241,6 +1243,7 @@ next_match:
 
                        scratch->map_index = map_index;
                        kernel_fpu_end();
+                       __local_unlock_nested_bh(&scratch->bh_lock);
                        return e;
                }
 
@@ -1250,6 +1253,7 @@ next_match:
        }
 
        kernel_fpu_end();
+       __local_unlock_nested_bh(&scratch->bh_lock);
        return NULL;
 }