sif: Add vendor flag to support testing without oversized CQs

After introduction of extra CQ entries to reduce risk of
having duplicate completions overflow a CQ, we no longer can
trigger various CQ overflow scenarios without running a lot of
requests. We need to be able to test with a minimal set of operations
to allow co-sim based tests for further analysis.

Introduce a new vendor_flag no_x_cqe = 0x80 to turn off
the allocation of extra CQEs.

Orabug: 24919301

Signed-off-by: Knut Omang <knut.omang@oracle.com>
Reviewed-by: Francisco Triviño <francisco.trivino@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c
index 631df0d70be206440f0352725e7ab6fae3108dce..2087ea67dc36de66b2fb26b42469a72e389ebdfe 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_cq.c
+++ b/drivers/infiniband/hw/sif/sif_cq.c
@@ -109,7 +109,7 @@ struct ib_cq *sif_create_cq(struct ib_device *ibdev, int entries,
                        user_mode = !user_mode;
        }
 
-       cq = create_cq(pd, entries, comp_vector, proxy, user_mode);
+       cq = create_cq(pd, entries, comp_vector, proxy, user_mode, user_flags);
        if (IS_ERR(cq))
                return (struct ib_cq *)cq;
 
@@ -136,7 +136,8 @@ struct ib_cq *sif_create_cq(struct ib_device *ibdev, int entries,
 struct sif_cq *create_cq(struct sif_pd *pd, int entries,
                        int comp_vector,
                        enum sif_proxy_type proxy,
-                       bool user_mode)
+                       bool user_mode,
+                       enum sif_vendor_flags user_flags)
 {
        struct sif_dev *sdev = to_sdev(pd->ibpd.device);
        struct sif_cq_sw *cq_sw;
@@ -148,6 +149,7 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries,
        u64 alloc_sz;
        int ret;
        int index = sif_alloc_cq_hw_idx(pd);
+       int headroom = 1;
 
        if (index < 0) {
                ecq = ERR_PTR(-ENOMEM);
@@ -173,10 +175,14 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries,
        /* Make sure we never fill the CQ completely on rev 1-3 - Bug #3657.
         * bug #4074 - add SIF_SW_RESERVED_DUL_CQE 768 entries to avoid worst case
         * duplicate completions that can be generated by HW and 1 entry for the
-        * fence completion.
+        * fence completion. (This extra headroom can be disabled for test purposes
+        * only using the no_x_cqe vendor_flag)
         */
+       if (!sif_vendor_enable(no_x_cqe, user_flags))
+               headroom = SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE;
+
        if (PSIF_REVISION(sdev) <= 3)
-               entries += (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE);
+               entries += headroom;
 
        cq->entries = roundup_pow_of_two(entries);
        cq->ibcq.cqe = cq->entries;
@@ -184,7 +190,7 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries,
 
        /* Adjust available cqes on rev 1-3 - Bug #3657 and #4074. */
        if (PSIF_REVISION(sdev) <= 3)
-               cq->ibcq.cqe -= (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE);
+               cq->ibcq.cqe -= headroom;
 
        /* See #2965: 5 bit size_log2 field in cq desc
         * but counter is 32 bit. For simplicity to distinguish full from empty

diff --git a/drivers/infiniband/hw/sif/sif_cq.h b/drivers/infiniband/hw/sif/sif_cq.h
index d911521bd2dc0ce3962f993641b03c72b3501137..d69113280ac8001eb9791595c003ae0f11037a89 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_cq.h
+++ b/drivers/infiniband/hw/sif/sif_cq.h
@@ -63,7 +63,8 @@ int poll_wait_for_cq_writeback(struct sif_dev *sdev, struct sif_cq *cq);
 struct sif_cq *create_cq(struct sif_pd *pd, int cqe,
                        int comp_vector,
                        enum sif_proxy_type proxy,
-                       bool user_mode);
+                       bool user_mode,
+                       enum sif_vendor_flags user_flags);
 
 
 /* internal poll/peek of completion queue:

diff --git a/drivers/infiniband/hw/sif/sif_pqp.c b/drivers/infiniband/hw/sif/sif_pqp.c
index 829095e77fb579d5f09023c38d6f9f2809658396..04f917c29c881e2414b48ff60a2ce37ac10808ee 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pqp.c
+++ b/drivers/infiniband/hw/sif/sif_pqp.c
@@ -76,7 +76,7 @@ static struct sif_pqp *_sif_create_pqp(struct sif_dev *sdev, size_t alloc_sz, in
                return NULL;
        }
 
-       cq = create_cq(sdev->pd, sif_max_pqp_wr, comp_vector, SIFPX_OFF, false);
+       cq = create_cq(sdev->pd, sif_max_pqp_wr, comp_vector, SIFPX_OFF, false, 0);
        if (IS_ERR(cq)) {
                ret = PTR_ERR(cq);
                goto cq_alloc_failed;

diff --git a/drivers/infiniband/hw/sif/sif_r3.c b/drivers/infiniband/hw/sif/sif_r3.c
index 06615b842c218aacdd039123aed55093391eec28..f7be78b83fbec31786d1a2816f5249ccbcc01fa8 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_r3.c
+++ b/drivers/infiniband/hw/sif/sif_r3.c
@@ -212,7 +212,7 @@ static int sif_hw_allocate_flush_qp(struct sif_dev *sdev, u8 flush_idx)
        /* CQ */
        cq = create_cq(sdev->pd,
                init_attr.cap.max_send_wr + init_attr.cap.max_recv_wr,
-               1, SIFPX_OFF, false);
+               1, SIFPX_OFF, false, 0);
        if (IS_ERR(cq)) {
                sif_log(sdev, SIF_INFO, "Failed to create CQ for flush_retry QP port %d", port);
                return -EINVAL;

diff --git a/drivers/infiniband/hw/sif/sif_user.h b/drivers/infiniband/hw/sif/sif_user.h
index 35c8af7793c9ed4d43ee41f71a2ec4561ed7baa0..1e62f4806c6326eb6ee213a0ec0b690ee17d664d 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_user.h
+++ b/drivers/infiniband/hw/sif/sif_user.h
@@ -65,6 +65,7 @@ enum sif_vendor_flags {
        tsu_qosl      = 0x10,   /* Value to use for the qosl bit in the qp state */
        no_checksum   = 0x20,   /* No csum for qp, wqe.wr.csum = qp.magic */
        dynamic_mtu   = 0x40,   /* dynamic MTU - use 256B instead of the path MTU */
+       no_x_cqe      = 0x80,   /* Don't allocate room in the CQs to avoid overflow due to #4074 */
 };
 
 enum sif_mem_type {