From: Wei Lin Guay Date: Thu, 30 Jun 2016 08:42:26 +0000 (+0200) Subject: sif: qp: added persistent_state in sif_qp struct X-Git-Tag: v4.1.12-92~129^2~4 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=43609bf97210f3036436fd137c88fd2bb3291e85;p=users%2Fjedix%2Flinux-maple.git sif: qp: added persistent_state in sif_qp struct Orabug: 23491094 QP state needs to be referred in certain context which may not sleep. Nevertheless, the state is not guarantee as mutex cannot be used. Thus, this commit added new atomic persistent_state to determine the QP state in non-sleep context. This commit removes non-used flush_sq_done_wa4074 variable and added mutex for sif_query_qp due to WA #3714 and WA #662. In SIF, there is intermediate QP state from RTS->ERR and RTS->RESET. Thus, without mutex, the sif_query_qp might gets the intermediate QP state. Signed-off-by: Wei Lin Guay Reviewed-by: Håkon Bugge Reviewed-by: Knut Omang --- diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c index 5ae29174c077..26270dee9e5b 100644 --- a/drivers/infiniband/hw/sif/sif_cq.c +++ b/drivers/infiniband/hw/sif/sif_cq.c @@ -462,8 +462,12 @@ static int handle_recv_wc(struct sif_dev *sdev, struct sif_cq *cq, struct ib_wc (wc->status != IB_WC_SUCCESS)) { struct sif_qp *qp = to_sqp(wc->qp); + /* As QP is in ERROR, the only scenario that + * rq shouldn't be flushed by SW is when the QP + * is in RESET state. + */ if (rq && !rq->is_srq - && IB_QPS_ERR == get_qp_state(qp)) { + && !test_bit(SIF_QPS_IN_RESET, &qp->persistent_state)) { if (sif_flush_rq(sdev, rq, qp, rq_len)) sif_log(sdev, SIF_INFO, "failed to flush RQ %d", rq->index); diff --git a/drivers/infiniband/hw/sif/sif_qp.c b/drivers/infiniband/hw/sif/sif_qp.c index 999b285c3c49..0e151111b9e5 100644 --- a/drivers/infiniband/hw/sif/sif_qp.c +++ b/drivers/infiniband/hw/sif/sif_qp.c @@ -463,6 +463,7 @@ struct sif_qp *create_qp(struct sif_dev *sdev, copy_conv_to_hw(&qp->d, &qpi, sizeof(struct psif_qp)); mutex_init(&qp->lock); /* TBD: Sync scheme! */ + set_bit(SIF_QPS_IN_RESET, &qp->persistent_state); /* Users should see qp 0/1 even though qp 0/1 is mapped to qp 2/3 for * port 2 @@ -478,7 +479,6 @@ struct sif_qp *create_qp(struct sif_dev *sdev, qp->ibqp.qp_type = IB_QPT_UD; } - qp->flush_sq_done_wa4074 = false; ret = sif_dfs_add_qp(sdev, qp); if (ret) @@ -1034,9 +1034,11 @@ sif_mqp_ret: */ switch (new_state) { case IB_QPS_RESET: + set_bit(SIF_QPS_IN_RESET, &qp->persistent_state); qp->flags &= ~SIF_QPF_HW_OWNED; break; case IB_QPS_RTR: + clear_bit(SIF_QPS_IN_RESET, &qp->persistent_state); qp->flags |= SIF_QPF_HW_OWNED; break; default: @@ -1814,6 +1816,10 @@ int sif_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, bool use_hw = false; struct sif_qp *qp = to_sqp(ibqp); struct sif_dev *sdev = to_sdev(ibqp->device); + int ret; + + /* Take QP lock to avoid any race condition on updates to last_set_state: */ + mutex_lock(&qp->lock); sif_logi(ibqp->device, SIF_QP, "last_set_state %d", qp->last_set_state); @@ -1826,33 +1832,21 @@ int sif_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, * ibv_query_qp might report wrong state when in state IBV_QPS_ERR * Query must be done based on current ownership (towards HW only if HW owned) */ - if (PSIF_REVISION(sdev) <= 3 || qp->flush_sq_done_wa4074) + if (PSIF_REVISION(sdev) <= 3) use_hw = (qp->flags & SIF_QPF_HW_OWNED); else use_hw = true; break; } - return use_hw ? + ret = use_hw ? sif_query_qp_hw(ibqp, qp_attr, qp_attr_mask, qp_init_attr) : sif_query_qp_sw(ibqp, qp_attr, qp_attr_mask, qp_init_attr); -} -enum ib_qp_state get_qp_state(struct sif_qp *qp) -{ - struct ib_qp *ibqp = &qp->ibqp; - struct ib_qp_init_attr init_attr; - struct ib_qp_attr attr; + mutex_unlock(&qp->lock); - memset(&attr, 0, sizeof(attr)); - memset(&init_attr, 0, sizeof(init_attr)); + return ret; - if (sif_query_qp(ibqp, &attr, IB_QP_STATE, &init_attr)) { - sif_logi(ibqp->device, SIF_INFO, - "query_qp failed for qp %d", ibqp->qp_num); - return -1; - } - return attr.qp_state; } static void get_qp_path_sw(struct sif_qp *qp, struct ib_qp_attr *qp_attr, bool alternate) @@ -2067,13 +2061,10 @@ static int sif_query_qp_hw(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, struct sif_sq *sq = get_sq(sdev, qp); struct psif_query_qp lqqp; - /* Take QP lock to avoid any race condition on updates to last_set_state: */ - mutex_lock(&qp->lock); ret = epsc_query_qp(qp, &lqqp); if (!ret) qp->last_set_state = sif2ib_qp_state(lqqp.qp.state); - mutex_unlock(&qp->lock); if (ret) return ret; @@ -2392,7 +2383,6 @@ failed: */ set_psif_qp_core__xmit_psn(&qps->state, 0); set_psif_qp_core__last_acked_psn(&qps->state, 0xffffff); - qp->flush_sq_done_wa4074 = false; return ret; } diff --git a/drivers/infiniband/hw/sif/sif_qp.h b/drivers/infiniband/hw/sif/sif_qp.h index 81ed291bdbf7..6f43dd6b032b 100644 --- a/drivers/infiniband/hw/sif/sif_qp.h +++ b/drivers/infiniband/hw/sif/sif_qp.h @@ -81,6 +81,11 @@ struct sif_qp_init_attr { int sq_hdl_sz; }; + +enum qp_persistent_state { + SIF_QPS_IN_RESET = 0, +}; + struct sif_qp { volatile struct psif_qp d; /* Hardware QPSC entry */ struct ib_qp ibqp ____cacheline_internodealigned_in_smp; @@ -124,8 +129,7 @@ struct sif_qp { int srq_idx; /* WA #3952: Track SRQ for modify_srq(used only for pQP) */ atomic64_t arm_srq_holdoff_time;/* Wait-time,if the pQP is held for a prev modify_srq */ - - bool flush_sq_done_wa4074; /* WA #4074: Track if QP state changes are already applied */ + unsigned long persistent_state; /* the atomic flag to determine the QP reset */ u64 ipoib_tx_csum_l3; u64 ipoib_tx_csum_l4; diff --git a/drivers/infiniband/hw/sif/sif_r3.c b/drivers/infiniband/hw/sif/sif_r3.c index 9ac7fa7a8357..2446fbc56916 100644 --- a/drivers/infiniband/hw/sif/sif_r3.c +++ b/drivers/infiniband/hw/sif/sif_r3.c @@ -738,7 +738,6 @@ err_post_wa4074: clear_bit(CQ_POLLING_NOT_ALLOWED, &cq_sw->flags); clear_bit(FLUSH_SQ_IN_FLIGHT, &sq_sw->flags); clear_bit(FLUSH_SQ_IN_PROGRESS, &sq_sw->flags); - qp->flush_sq_done_wa4074 = true; return ret = ret > 0 ? 0 : ret; }