From 9bcfbc52475bb2647af099e50a62f9b9bf9039ef Mon Sep 17 00:00:00 2001 From: Knut Omang Date: Thu, 13 Oct 2016 12:07:33 +0200 Subject: [PATCH] sif: Add vendor flag to support testing without oversized CQs MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit After introduction of extra CQ entries to reduce risk of having duplicate completions overflow a CQ, we no longer can trigger various CQ overflow scenarios without running a lot of requests. We need to be able to test with a minimal set of operations to allow co-sim based tests for further analysis. Introduce a new vendor_flag no_x_cqe = 0x80 to turn off the allocation of extra CQEs. Orabug: 24919301 Signed-off-by: Knut Omang Reviewed-by: Francisco Triviño --- drivers/infiniband/hw/sif/sif_cq.c | 16 +++++++++++----- drivers/infiniband/hw/sif/sif_cq.h | 3 ++- drivers/infiniband/hw/sif/sif_pqp.c | 2 +- drivers/infiniband/hw/sif/sif_r3.c | 2 +- drivers/infiniband/hw/sif/sif_user.h | 1 + 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c index 631df0d70be2..2087ea67dc36 100644 --- a/drivers/infiniband/hw/sif/sif_cq.c +++ b/drivers/infiniband/hw/sif/sif_cq.c @@ -109,7 +109,7 @@ struct ib_cq *sif_create_cq(struct ib_device *ibdev, int entries, user_mode = !user_mode; } - cq = create_cq(pd, entries, comp_vector, proxy, user_mode); + cq = create_cq(pd, entries, comp_vector, proxy, user_mode, user_flags); if (IS_ERR(cq)) return (struct ib_cq *)cq; @@ -136,7 +136,8 @@ struct ib_cq *sif_create_cq(struct ib_device *ibdev, int entries, struct sif_cq *create_cq(struct sif_pd *pd, int entries, int comp_vector, enum sif_proxy_type proxy, - bool user_mode) + bool user_mode, + enum sif_vendor_flags user_flags) { struct sif_dev *sdev = to_sdev(pd->ibpd.device); struct sif_cq_sw *cq_sw; @@ -148,6 +149,7 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries, u64 alloc_sz; int ret; int index = sif_alloc_cq_hw_idx(pd); + int headroom = 1; if (index < 0) { ecq = ERR_PTR(-ENOMEM); @@ -173,10 +175,14 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries, /* Make sure we never fill the CQ completely on rev 1-3 - Bug #3657. * bug #4074 - add SIF_SW_RESERVED_DUL_CQE 768 entries to avoid worst case * duplicate completions that can be generated by HW and 1 entry for the - * fence completion. + * fence completion. (This extra headroom can be disabled for test purposes + * only using the no_x_cqe vendor_flag) */ + if (!sif_vendor_enable(no_x_cqe, user_flags)) + headroom = SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE; + if (PSIF_REVISION(sdev) <= 3) - entries += (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE); + entries += headroom; cq->entries = roundup_pow_of_two(entries); cq->ibcq.cqe = cq->entries; @@ -184,7 +190,7 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries, /* Adjust available cqes on rev 1-3 - Bug #3657 and #4074. */ if (PSIF_REVISION(sdev) <= 3) - cq->ibcq.cqe -= (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE); + cq->ibcq.cqe -= headroom; /* See #2965: 5 bit size_log2 field in cq desc * but counter is 32 bit. For simplicity to distinguish full from empty diff --git a/drivers/infiniband/hw/sif/sif_cq.h b/drivers/infiniband/hw/sif/sif_cq.h index d911521bd2dc..d69113280ac8 100644 --- a/drivers/infiniband/hw/sif/sif_cq.h +++ b/drivers/infiniband/hw/sif/sif_cq.h @@ -63,7 +63,8 @@ int poll_wait_for_cq_writeback(struct sif_dev *sdev, struct sif_cq *cq); struct sif_cq *create_cq(struct sif_pd *pd, int cqe, int comp_vector, enum sif_proxy_type proxy, - bool user_mode); + bool user_mode, + enum sif_vendor_flags user_flags); /* internal poll/peek of completion queue: diff --git a/drivers/infiniband/hw/sif/sif_pqp.c b/drivers/infiniband/hw/sif/sif_pqp.c index 829095e77fb5..04f917c29c88 100644 --- a/drivers/infiniband/hw/sif/sif_pqp.c +++ b/drivers/infiniband/hw/sif/sif_pqp.c @@ -76,7 +76,7 @@ static struct sif_pqp *_sif_create_pqp(struct sif_dev *sdev, size_t alloc_sz, in return NULL; } - cq = create_cq(sdev->pd, sif_max_pqp_wr, comp_vector, SIFPX_OFF, false); + cq = create_cq(sdev->pd, sif_max_pqp_wr, comp_vector, SIFPX_OFF, false, 0); if (IS_ERR(cq)) { ret = PTR_ERR(cq); goto cq_alloc_failed; diff --git a/drivers/infiniband/hw/sif/sif_r3.c b/drivers/infiniband/hw/sif/sif_r3.c index 06615b842c21..f7be78b83fbe 100644 --- a/drivers/infiniband/hw/sif/sif_r3.c +++ b/drivers/infiniband/hw/sif/sif_r3.c @@ -212,7 +212,7 @@ static int sif_hw_allocate_flush_qp(struct sif_dev *sdev, u8 flush_idx) /* CQ */ cq = create_cq(sdev->pd, init_attr.cap.max_send_wr + init_attr.cap.max_recv_wr, - 1, SIFPX_OFF, false); + 1, SIFPX_OFF, false, 0); if (IS_ERR(cq)) { sif_log(sdev, SIF_INFO, "Failed to create CQ for flush_retry QP port %d", port); return -EINVAL; diff --git a/drivers/infiniband/hw/sif/sif_user.h b/drivers/infiniband/hw/sif/sif_user.h index 35c8af7793c9..1e62f4806c63 100644 --- a/drivers/infiniband/hw/sif/sif_user.h +++ b/drivers/infiniband/hw/sif/sif_user.h @@ -65,6 +65,7 @@ enum sif_vendor_flags { tsu_qosl = 0x10, /* Value to use for the qosl bit in the qp state */ no_checksum = 0x20, /* No csum for qp, wqe.wr.csum = qp.magic */ dynamic_mtu = 0x40, /* dynamic MTU - use 256B instead of the path MTU */ + no_x_cqe = 0x80, /* Don't allocate room in the CQs to avoid overflow due to #4074 */ }; enum sif_mem_type { -- 2.50.1