sif: qp: added persistent_state in sif_qp struct

Orabug: 23491094

QP state needs to be referred in certain context which may not sleep.
Nevertheless, the state is not guarantee as mutex cannot be used. Thus,
this commit added new atomic persistent_state to determine the QP
state in non-sleep context.

This commit removes non-used flush_sq_done_wa4074 variable and added
mutex for sif_query_qp due to WA #3714 and WA #662. In SIF, there is
intermediate QP state from RTS->ERR and RTS->RESET. Thus, without
mutex, the sif_query_qp might gets the intermediate QP state.

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>
Reviewed-by: Knut Omang <knut.omang@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c
index 5ae29174c077be587744c035afb358318447b97a..26270dee9e5be3007ad1e80ed46c7303da8ee077 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_cq.c
+++ b/drivers/infiniband/hw/sif/sif_cq.c
@@ -462,8 +462,12 @@ static int handle_recv_wc(struct sif_dev *sdev, struct sif_cq *cq, struct ib_wc
                (wc->status != IB_WC_SUCCESS)) {
                struct sif_qp *qp = to_sqp(wc->qp);
 
+               /* As QP is in ERROR, the only scenario that
+                * rq shouldn't be flushed by SW is when the QP
+                * is in RESET state.
+                */
                if (rq && !rq->is_srq
-                       && IB_QPS_ERR == get_qp_state(qp)) {
+                   && !test_bit(SIF_QPS_IN_RESET, &qp->persistent_state)) {
                        if (sif_flush_rq(sdev, rq, qp, rq_len))
                                sif_log(sdev, SIF_INFO,
                                        "failed to flush RQ %d", rq->index);

diff --git a/drivers/infiniband/hw/sif/sif_qp.c b/drivers/infiniband/hw/sif/sif_qp.c
index 999b285c3c497b319c5d3fad73205a7791f762b4..0e151111b9e5fe045077e284cf49bfe8914a09c7 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_qp.c
+++ b/drivers/infiniband/hw/sif/sif_qp.c
@@ -463,6 +463,7 @@ struct sif_qp *create_qp(struct sif_dev *sdev,
        copy_conv_to_hw(&qp->d, &qpi, sizeof(struct psif_qp));
 
        mutex_init(&qp->lock); /* TBD: Sync scheme! */
+       set_bit(SIF_QPS_IN_RESET, &qp->persistent_state);
 
        /* Users should see qp 0/1 even though qp 0/1 is mapped to qp 2/3 for
         * port 2
@@ -478,7 +479,6 @@ struct sif_qp *create_qp(struct sif_dev *sdev,
                qp->ibqp.qp_type = IB_QPT_UD;
        }
 
-       qp->flush_sq_done_wa4074 = false;
 
        ret = sif_dfs_add_qp(sdev, qp);
        if (ret)
@@ -1034,9 +1034,11 @@ sif_mqp_ret:
         */
        switch (new_state) {
        case IB_QPS_RESET:
+               set_bit(SIF_QPS_IN_RESET, &qp->persistent_state);
                qp->flags &= ~SIF_QPF_HW_OWNED;
                break;
        case IB_QPS_RTR:
+               clear_bit(SIF_QPS_IN_RESET, &qp->persistent_state);
                qp->flags |= SIF_QPF_HW_OWNED;
                break;
        default:
@@ -1814,6 +1816,10 @@ int sif_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
        bool use_hw = false;
        struct sif_qp *qp = to_sqp(ibqp);
        struct sif_dev *sdev = to_sdev(ibqp->device);
+       int ret;
+
+       /* Take QP lock to avoid any race condition on updates to last_set_state: */
+       mutex_lock(&qp->lock);
 
        sif_logi(ibqp->device, SIF_QP, "last_set_state %d", qp->last_set_state);
 
@@ -1826,33 +1832,21 @@ int sif_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
                 * ibv_query_qp might report wrong state when in state IBV_QPS_ERR
                 * Query must be done based on current ownership (towards HW only if HW owned)
                 */
-               if (PSIF_REVISION(sdev) <= 3 || qp->flush_sq_done_wa4074)
+               if (PSIF_REVISION(sdev) <= 3)
                        use_hw = (qp->flags & SIF_QPF_HW_OWNED);
                else
                        use_hw = true;
                break;
        }
 
-       return use_hw ?
+       ret = use_hw ?
                sif_query_qp_hw(ibqp, qp_attr, qp_attr_mask, qp_init_attr) :
                sif_query_qp_sw(ibqp, qp_attr, qp_attr_mask, qp_init_attr);
-}
 
-enum ib_qp_state get_qp_state(struct sif_qp *qp)
-{
-       struct ib_qp *ibqp = &qp->ibqp;
-       struct ib_qp_init_attr init_attr;
-       struct ib_qp_attr attr;
+       mutex_unlock(&qp->lock);
 
-       memset(&attr, 0, sizeof(attr));
-       memset(&init_attr, 0, sizeof(init_attr));
+       return ret;
 
-       if (sif_query_qp(ibqp, &attr, IB_QP_STATE, &init_attr)) {
-               sif_logi(ibqp->device, SIF_INFO,
-                       "query_qp failed for qp %d", ibqp->qp_num);
-               return -1;
-       }
-       return attr.qp_state;
 }
 
 static void get_qp_path_sw(struct sif_qp *qp, struct ib_qp_attr *qp_attr, bool alternate)
@@ -2067,13 +2061,10 @@ static int sif_query_qp_hw(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
        struct sif_sq *sq = get_sq(sdev, qp);
        struct psif_query_qp lqqp;
 
-       /* Take QP lock to avoid any race condition on updates to last_set_state: */
-       mutex_lock(&qp->lock);
 
        ret = epsc_query_qp(qp, &lqqp);
        if (!ret)
                qp->last_set_state = sif2ib_qp_state(lqqp.qp.state);
-       mutex_unlock(&qp->lock);
 
        if (ret)
                return ret;
@@ -2392,7 +2383,6 @@ failed:
         */
        set_psif_qp_core__xmit_psn(&qps->state, 0);
        set_psif_qp_core__last_acked_psn(&qps->state, 0xffffff);
-       qp->flush_sq_done_wa4074 = false;
 
        return ret;
 }

diff --git a/drivers/infiniband/hw/sif/sif_qp.h b/drivers/infiniband/hw/sif/sif_qp.h
index 81ed291bdbf73e8d9270e8f3571c07392c14bc1a..6f43dd6b032bfefe1c4d4bc3d9940038453df7b9 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_qp.h
+++ b/drivers/infiniband/hw/sif/sif_qp.h
@@ -81,6 +81,11 @@ struct sif_qp_init_attr {
        int sq_hdl_sz;
 };
 
+
+enum qp_persistent_state {
+       SIF_QPS_IN_RESET  = 0,
+};
+
 struct sif_qp {
        volatile struct psif_qp d;      /* Hardware QPSC entry */
        struct ib_qp ibqp ____cacheline_internodealigned_in_smp;
@@ -124,8 +129,7 @@ struct sif_qp {
 
        int srq_idx;                    /* WA #3952: Track SRQ for modify_srq(used only for pQP) */
        atomic64_t arm_srq_holdoff_time;/* Wait-time,if the pQP is held for a prev modify_srq */
-
-       bool flush_sq_done_wa4074;      /* WA #4074: Track if QP state changes are already applied */
+       unsigned long persistent_state; /* the atomic flag to determine the QP reset */
 
        u64 ipoib_tx_csum_l3;
        u64 ipoib_tx_csum_l4;

diff --git a/drivers/infiniband/hw/sif/sif_r3.c b/drivers/infiniband/hw/sif/sif_r3.c
index 9ac7fa7a83572322ec54c75f3dbc6d887ccedc55..2446fbc56916968555d0bf452cea1348c3007005 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_r3.c
+++ b/drivers/infiniband/hw/sif/sif_r3.c
@@ -738,7 +738,6 @@ err_post_wa4074:
        clear_bit(CQ_POLLING_NOT_ALLOWED, &cq_sw->flags);
        clear_bit(FLUSH_SQ_IN_FLIGHT, &sq_sw->flags);
        clear_bit(FLUSH_SQ_IN_PROGRESS, &sq_sw->flags);
-       qp->flush_sq_done_wa4074 = true;
        return ret = ret > 0 ? 0 : ret;
 }