net/mlx5e: Prepare for using different CQ doorbells

Completion queues (CQs) in mlx5 use the same global doorbell, which may
become contended when accessed concurrently from many cores.

This patch prepares the CQ management code for supporting different
doorbells per CQ. This will be used in downstream patches to allow
separate doorbells to be used by channels CQs.

The main change is moving the 'uar' pointer from struct mlx5_core_cq to
struct mlx5e_cq, as the uar page to be used is better off stored
directly there. Other users of mlx5_core_cq also store the UAR to be
used separately and therefore the pointer being removed is dead weight
for them. As evidence, in this patch there are two users which set the
mcq.uar pointer but didn't use it, Software Steering and old Innova CQ
creation code. Instead, they rang the doorbell directly from another
pointer.

The 'uar' pointer added to struct mlx5e_cq remains in a hot cacheline
(as before), because it may get accessed for each packet.

Signed-off-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
index 35039a95dcfd43697208a959733b4a7afdadfa11..e9f319a9bdd6be4294068a22de746cadf999f141 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
@@ -145,7 +145,6 @@ int mlx5_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
                mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n",
                              cq->cqn);
 
-       cq->uar = dev->priv.bfreg.up;
        cq->irqn = eq->core.irqn;
 
        return 0;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 9c73165653bff182c1e521cf49bf6277d68df5c7..1cbe3f3037bb2c911fe9870c2c0381659d409436 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -344,6 +344,7 @@ struct mlx5e_cq {
        /* data path - accessed per napi poll */
        u16                        event_ctr;
        struct napi_struct        *napi;
+       struct mlx5_uars_page     *uar;
        struct mlx5_core_cq        mcq;
        struct mlx5e_ch_stats     *ch_stats;
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
index 5dc04bbfc71bbad4ca884b2ea994749d611be1d8..6760bb0336df914baad55bafa66ed7a9996aa0da 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
@@ -309,10 +309,7 @@ mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
 
 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
 {
-       struct mlx5_core_cq *mcq;
-
-       mcq = &cq->mcq;
-       mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
+       mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, cq->uar->map, cq->wq.cc);
 }
 
 static inline struct mlx5e_sq_dma *

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index c1491ed3db1cbc15556ea7842ebc528eb1068bab..93505c82d6f0118af6af930b51edf0ab26721619 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -2189,6 +2189,7 @@ static void mlx5e_close_xdpredirect_sq(struct mlx5e_xdpsq *xdpsq)
 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
                                 struct net_device *netdev,
                                 struct workqueue_struct *workqueue,
+                                struct mlx5_uars_page *uar,
                                 struct mlx5e_cq_param *param,
                                 struct mlx5e_cq *cq)
 {
@@ -2220,6 +2221,7 @@ static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
        cq->mdev = mdev;
        cq->netdev = netdev;
        cq->workqueue = workqueue;
+       cq->uar = uar;
 
        return 0;
 }
@@ -2235,7 +2237,8 @@ static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev,
        param->wq.db_numa_node  = ccp->node;
        param->eq_ix            = ccp->ix;
 
-       err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, param, cq);
+       err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq,
+                                   mdev->priv.bfreg.up, param, cq);
 
        cq->napi     = ccp->napi;
        cq->ch_stats = ccp->ch_stats;
@@ -2280,7 +2283,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
        MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode));
 
        MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
-       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
+       MLX5_SET(cqc,   cqc, uar_page,      cq->uar->index);
        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
                                            MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
@@ -3593,7 +3596,8 @@ static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
        param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
        param->wq.db_numa_node  = dev_to_node(mlx5_core_dma_dev(mdev));
 
-       return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, param, cq);
+       return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq,
+                                    mdev->priv.bfreg.up, param, cq);
 }
 
 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
index c4de6bf8d1b65d9153fb5634ad7b3a310eb61311..cb1319974f83f94d18fde978ba0df6759ad4d6eb 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
@@ -475,7 +475,6 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
        *conn->cq.mcq.arm_db    = 0;
        conn->cq.mcq.vector     = 0;
        conn->cq.mcq.comp       = mlx5_fpga_conn_cq_complete;
-       conn->cq.mcq.uar        = fdev->conn_res.uar;
        tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet);
 
        mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c
index 4fd4e8483382cc80579d19f599153d3f0b1040bb..077a77fde670ef0ccc80ea1776699b523ce7647d 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_send.c
@@ -1131,7 +1131,6 @@ static struct mlx5dr_cq *dr_create_cq(struct mlx5_core_dev *mdev,
        *cq->mcq.arm_db = cpu_to_be32(2 << 28);
 
        cq->mcq.vector = 0;
-       cq->mcq.uar = uar;
        cq->mdev = mdev;
 
        return cq;

diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h
index 991526039ccbd6fa2009dafea8c355b0f45abf20..7ef2c7c7d803ded6c52ab700304fc50e9c2a3ad5 100644 (file)
--- a/include/linux/mlx5/cq.h
+++ b/include/linux/mlx5/cq.h
@@ -41,7 +41,6 @@ struct mlx5_core_cq {
        int                     cqe_sz;
        __be32                 *set_ci_db;
        __be32                 *arm_db;
-       struct mlx5_uars_page  *uar;
        refcount_t              refcount;
        struct completion       free;
        unsigned                vector;