sif: cq: fixup the CQEs when a QP is transitioned to RESET

Orabug: 24652927

The sif_fixup_cqes function, which update the wr_id from the SQ handle, is
moved to reset_qp to cover the scenario where IB user reuses a QP after
performing ib_modify_qp(RESET). This patch also handles a scenario in
sif_fixup_cqes where a QP has been reset multiple times but the IB user has
not polled the associated CQ completely.

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Reviewed-by: Knut Omang <knut.omang@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c
index 6dda1da04fbb9fb4d629bc6cd18245eb588a97d4..04bb791f742130e505a129e4329c60dd0d1e8dbc 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_cq.c
+++ b/drivers/infiniband/hw/sif/sif_cq.c
@@ -780,6 +780,12 @@ int sif_fixup_cqes(struct sif_cq *cq, struct sif_sq *sq, struct sif_qp *qp)
                /* Read into local copy in host memory order */
                copy_conv_to_sw(&lcqe, cqe, sizeof(lcqe));
 
+               /* In a scenario where QP has been destroyed/reset multiple times but the
+                * associated CQ has not being polled yet.
+                */
+               if (lcqe.opcode & SIF_WC_QP_DESTROYED)
+                       continue;
+
                /* Receive completion? */
                if (lcqe.opcode & PSIF_WC_OPCODE_RECEIVE_SEND) {
                        struct sif_post_mortem_qp_info_in_cqe *post_mortem_info =
@@ -790,6 +796,7 @@ int sif_fixup_cqes(struct sif_cq *cq, struct sif_sq *sq, struct sif_qp *qp)
                        post_mortem_info->srq_idx = qp->rq_idx;
                        post_mortem_info->qpn     = qp->qp_idx;
                } else {
+
                        /* If a send completion, handle the wr_id */
                        ret = translate_wr_id(&wr_id_host_order, sdev, cq, sq, &lcqe,
                                        lcqe.wc_id.sq_id.sq_seq_num, lcqe.qp);

diff --git a/drivers/infiniband/hw/sif/sif_qp.c b/drivers/infiniband/hw/sif/sif_qp.c
index e448701467dfec1cec45c93ba48d407630148722..0eb2e90a610da563bcfab5f8c5ee48ae7273edb9 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_qp.c
+++ b/drivers/infiniband/hw/sif/sif_qp.c
@@ -2203,47 +2203,6 @@ int destroy_qp(struct sif_dev *sdev, struct sif_qp *qp)
        if (ret)
                sif_log(sdev, SIF_INFO, "modify qp %d to RESET failed, sts %d", index, ret);
 
-       if (!(qp->flags & SIF_QPF_USER_MODE)) {
-               int nfixup;
-               struct sif_sq *sq = get_sq(sdev, qp);
-               u32 cq_idx = get_psif_qp_core__rcv_cq_indx(&qp->d.state);
-               struct sif_cq *send_cq = (sq && sq->cq_idx >= 0) ? get_sif_cq(sdev, sq->cq_idx) : NULL;
-               struct sif_cq *recv_cq = rq ? get_sif_cq(sdev, cq_idx) : NULL;
-
-               if (send_cq) {
-                       ret = post_process_wa4074(sdev, qp);
-                       if (ret) {
-                               sif_log(sdev, SIF_INFO,
-                                       "post_process_wa4074 failed for qp %d send cq %d with error %d",
-                                       qp->qp_idx, sq->cq_idx, ret);
-                               goto fixup_failed;
-                       }
-
-                       nfixup = sif_fixup_cqes(send_cq, sq, qp);
-                       if (nfixup < 0) {
-                               sif_log(sdev, SIF_INFO,
-                                       "fixup cqes on qp %d send cq %d failed with error %d",
-                                       qp->qp_idx, sq->cq_idx, nfixup);
-                               goto fixup_failed;
-                       }
-                       sif_log(sdev, SIF_QP, "fixup cqes fixed %d CQEs in sq.cq %d",
-                               nfixup, sq->cq_idx);
-               }
-               if (recv_cq && recv_cq != send_cq) {
-                       nfixup = sif_fixup_cqes(recv_cq, sq, qp);
-                       if (nfixup < 0) {
-                               sif_log(sdev, SIF_INFO,
-                                       "fixup cqes on qp %d recv cq %d failed with error %d",
-                                       qp->qp_idx, cq_idx, nfixup);
-                               goto fixup_failed;
-                       }
-                       sif_log(sdev, SIF_QP, "fixup cqes fixed %d CQEs in rq.cq %d",
-                               nfixup, cq_idx);
-
-               }
-       }
-
-fixup_failed:
        if (qp->qp_idx < 4) {
                /* Special QP cleanup */
                int ok = atomic_add_unless(&sdev->sqp_usecnt[qp->qp_idx], -1, 0);
@@ -2306,7 +2265,40 @@ static int reset_qp(struct sif_dev *sdev, struct sif_qp *qp)
 
        }
 
+       if (!(qp->flags & SIF_QPF_USER_MODE)) {
+               int nfixup;
+               u32 cq_idx = get_psif_qp_core__rcv_cq_indx(&qp->d.state);
+               struct sif_cq *send_cq = (sq && sq->cq_idx >= 0) ? get_sif_cq(sdev, sq->cq_idx) : NULL;
+               struct sif_cq *recv_cq = rq ? get_sif_cq(sdev, cq_idx) : NULL;
+
 
+               /* clean-up the SQ/RQ CQ before reset the SQ */
+               if (send_cq) {
+                       nfixup = sif_fixup_cqes(send_cq, sq, qp);
+                       if (nfixup < 0) {
+                               sif_log(sdev, SIF_INFO,
+                                       "fixup cqes on qp %d send cq %d failed with error %d",
+                                       qp->qp_idx, sq->cq_idx, nfixup);
+                               goto fixup_failed;
+                       }
+                       sif_log(sdev, SIF_QP, "fixup cqes fixed %d CQEs in sq.cq %d",
+                               nfixup, sq->cq_idx);
+               }
+               if (recv_cq && recv_cq != send_cq) {
+                       nfixup = sif_fixup_cqes(recv_cq, sq, qp);
+                       if (nfixup < 0) {
+                               sif_log(sdev, SIF_INFO,
+                                       "fixup cqes on qp %d recv cq %d failed with error %d",
+                                       qp->qp_idx, cq_idx, nfixup);
+                               goto fixup_failed;
+                       }
+                       sif_log(sdev, SIF_QP, "fixup cqes fixed %d CQEs in rq.cq %d",
+                               nfixup, cq_idx);
+
+               }
+       }
+
+fixup_failed:
        /* if the send queue scheduler is running, wait for
         * it to terminate:
         */