sif: cq: Add additional SIF visible cqes to CQ

Orabug: 24673784

Due to two HW bugs, SIF needs to add additional cqes apart from
the requested N cqes. First issue is that the CQ cannot be full
when it is being invalidated. Hence, we need 1 extra entry.

The second issue is that HW might generate duplicate completions.
Thus, SIF needs 768 extra entries to cater for these duplicate
completions and 1 additional entry for the fence completion.

Then, SIF driver rounds up to the nearest 2^N. The number of cqes
available to the ulp/user, will be the above 2^N - (768 + 1 + 1).

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_cq.c b/drivers/infiniband/hw/sif/sif_cq.c
index a2cd5e559343113db6bf577a3f7a8d598d7b4a53..4d0eb3693f4afec42b35ba1aeec9b96a64f22b49 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_cq.c
+++ b/drivers/infiniband/hw/sif/sif_cq.c
@@ -62,7 +62,7 @@ static inline int translate_wr_id(
                return -EFAULT;
        }
        if (unlikely(wh->sq_seq != sq_seq_num)) {
-               bool dup_case  = (wh->sq_seq - sq_seq_num == sq->entries);
+               bool dup_case = (wh->sq_seq - sq_seq_num == sq->entries);
                int log_level = dup_case ? SIF_WCE : SIF_INFO;
 
                sif_log(sdev, log_level,
@@ -170,17 +170,21 @@ struct sif_cq *create_cq(struct sif_pd *pd, int entries,
        cq_sw->next_seq = 0;
        cq_sw->last_hw_seq = 0;
 
-       /* Make sure we never fill the CQ completely on rev 1-3 - Bug #3657 */
+       /* Make sure we never fill the CQ completely on rev 1-3 - Bug #3657.
+        * bug #4074 - add SIF_SW_RESERVED_DUL_CQE 768 entries to avoid worst case
+        * duplicate completions that can be generated by HW and 1 entry for the
+        * fence completion.
+        */
        if (PSIF_REVISION(sdev) <= 3)
-               entries++;
+               entries += (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE);
 
        cq->entries = roundup_pow_of_two(entries);
        cq->ibcq.cqe = cq->entries;
        entries_log2 = order_base_2(cq->entries);
 
-       /* Adjust available cqes on rev 1-3 - Bug #3657 */
+       /* Adjust available cqes on rev 1-3 - Bug #3657 and #4074. */
        if (PSIF_REVISION(sdev) <= 3)
-               cq->ibcq.cqe--;
+               cq->ibcq.cqe -= (SIF_SW_RESERVED_DUL_CQE + SIF_SW_RESERVED_LAST_CQE);
 
        /* See #2965: 5 bit size_log2 field in cq desc
         * but counter is 32 bit. For simplicity to distinguish full from empty

diff --git a/drivers/infiniband/hw/sif/sif_dev.h b/drivers/infiniband/hw/sif/sif_dev.h
index c7092c9e9318f4cfaa707e13af98889451e070d6..fafe70e220227b2973501b4f94fc50666dba96bc 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_dev.h
+++ b/drivers/infiniband/hw/sif/sif_dev.h
@@ -69,6 +69,9 @@
  */
 #define SIF_SW_MAX_CQE_LOG2 0x18  /* = 16 MB - tested and should cover most use cases.. */
 #define SIF_SW_MAX_CQE (1 << SIF_SW_MAX_CQE_LOG2)
+/* 768 entries for the HW duplicate completions and 1 entry for the fence completion. */
+#define SIF_SW_RESERVED_DUL_CQE (768 + 1)
+#define SIF_SW_RESERVED_LAST_CQE 1
 
 #define SIF_SW_MAX_SQE_LOG2 0xf  /* = 32K */
 #define SIF_SW_MAX_SQE (1 << SIF_SW_MAX_SQE_LOG2)

diff --git a/drivers/infiniband/hw/sif/sif_user.h b/drivers/infiniband/hw/sif/sif_user.h
index a789323eea9fbbce87a01540c56f27763f1cbf81..35c8af7793c9ed4d43ee41f71a2ec4561ed7baa0 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_user.h
+++ b/drivers/infiniband/hw/sif/sif_user.h
@@ -21,7 +21,7 @@
  *
  */
 #define SIF_UVERBS_ABI_MAJOR_VERSION    3
-#define SIF_UVERBS_ABI_MINOR_VERSION    5
+#define SIF_UVERBS_ABI_MINOR_VERSION    6
 
 #define SIF_UVERBS_VERSION(x, y) ((x) << 8 | (y))