net/mlx5e: xsk: Improve need_wakeup logic

XSK need_wakeup mechanism allows the driver to stop busy waiting for
buffers when the fill ring is empty, yield to the application and signal
it that the driver needs to be waken up after the application refills
the fill ring.

Add protection against the race condition on the RX (refill) side: if
the application refills buffers after xskrq->post_wqes is called, but
before mlx5e_xsk_update_rx_wakeup, NAPI will exit, skipping taking these
buffers to the hardware WQ, and the application won't wake it up again.

Optimize the whole need_wakeup logic, removing unneeded flows, to
compensate for this new check.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
index 84a496a8d72fc55581cc5ad63c47c11b6b59b3e7..087c943bd8e90544c43d449765eac79a9cc42d4b 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -5,7 +5,6 @@
 #define __MLX5_EN_XSK_RX_H__
 
 #include "en.h"
-#include <net/xdp_sock_drv.h>
 
 /* RX data path */
 
@@ -21,17 +20,4 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
                                              struct mlx5e_wqe_frag_info *wi,
                                              u32 cqe_bcnt);
 
-static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err)
-{
-       if (!xsk_uses_need_wakeup(rq->xsk_pool))
-               return alloc_err;
-
-       if (unlikely(alloc_err))
-               xsk_set_rx_need_wakeup(rq->xsk_pool);
-       else
-               xsk_clear_rx_need_wakeup(rq->xsk_pool);
-
-       return false;
-}
-
 #endif /* __MLX5_EN_XSK_RX_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
index a05085035f23515bb6a113a0475f2b9ff79931e3..9c505158b975b5ac0063c70d511d6753dbf136b4 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
@@ -5,7 +5,6 @@
 #define __MLX5_EN_XSK_TX_H__
 
 #include "en.h"
-#include <net/xdp_sock_drv.h>
 
 /* TX data path */
 
@@ -13,15 +12,4 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
 
 bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget);
 
-static inline void mlx5e_xsk_update_tx_wakeup(struct mlx5e_xdpsq *sq)
-{
-       if (!xsk_uses_need_wakeup(sq->xsk_pool))
-               return;
-
-       if (sq->pc != sq->cc)
-               xsk_clear_tx_need_wakeup(sq->xsk_pool);
-       else
-               xsk_set_tx_need_wakeup(sq->xsk_pool);
-}
-
 #endif /* __MLX5_EN_XSK_TX_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 5835d86be8d865e1c75b6e4ea773ce2ec4864ec0..b61604d8770193dd52c46391ca026e2f0780be87 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -41,6 +41,7 @@
 #include <net/gro.h>
 #include <net/udp.h>
 #include <net/tcp.h>
+#include <net/xdp_sock_drv.h>
 #include "en.h"
 #include "en/txrx.h"
 #include "en_tc.h"

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 833be29170a135c54ae011e9d959279564ccf3f6..9a458a5d985394fcf5c9963278939b3a1e94246f 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/irq.h>
+#include <net/xdp_sock_drv.h>
 #include "en.h"
 #include "en/txrx.h"
 #include "en/xdp.h"
@@ -86,26 +87,36 @@ void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
 
 static bool mlx5e_napi_xsk_post(struct mlx5e_xdpsq *xsksq, struct mlx5e_rq *xskrq)
 {
+       bool need_wakeup = xsk_uses_need_wakeup(xskrq->xsk_pool);
        bool busy_xsk = false, xsk_rx_alloc_err;
 
-       /* Handle the race between the application querying need_wakeup and the
-        * driver setting it:
-        * 1. Update need_wakeup both before and after the TX. If it goes to
-        * "yes", it can only happen with the first update.
-        * 2. If the application queried need_wakeup before we set it, the
-        * packets will be transmitted anyway, even w/o a wakeup.
-        * 3. Give a chance to clear need_wakeup after new packets were queued
-        * for TX.
+       /* If SQ is empty, there are no TX completions to trigger NAPI, so set
+        * need_wakeup. Do it before queuing packets for TX to avoid race
+        * condition with userspace.
         */
-       mlx5e_xsk_update_tx_wakeup(xsksq);
+       if (need_wakeup && xsksq->pc == xsksq->cc)
+               xsk_set_tx_need_wakeup(xsksq->xsk_pool);
        busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET);
-       mlx5e_xsk_update_tx_wakeup(xsksq);
+       /* If we queued some packets for TX, no need for wakeup anymore. */
+       if (need_wakeup && xsksq->pc != xsksq->cc)
+               xsk_clear_tx_need_wakeup(xsksq->xsk_pool);
 
+       /* If WQ is empty, RX won't trigger NAPI, so set need_wakeup. Do it
+        * before refilling to avoid race condition with userspace.
+        */
+       if (need_wakeup && !mlx5e_rqwq_get_cur_sz(xskrq))
+               xsk_set_rx_need_wakeup(xskrq->xsk_pool);
        xsk_rx_alloc_err = INDIRECT_CALL_2(xskrq->post_wqes,
                                           mlx5e_post_rx_mpwqes,
                                           mlx5e_post_rx_wqes,
                                           xskrq);
-       busy_xsk |= mlx5e_xsk_update_rx_wakeup(xskrq, xsk_rx_alloc_err);
+       /* Ask for wakeup if WQ is not full after refill. */
+       if (!need_wakeup)
+               busy_xsk |= xsk_rx_alloc_err;
+       else if (xsk_rx_alloc_err)
+               xsk_set_rx_need_wakeup(xskrq->xsk_pool);
+       else
+               xsk_clear_rx_need_wakeup(xskrq->xsk_pool);
 
        return busy_xsk;
 }