sif: cb: Improve algorithm for allocating and using CBs from driver

Instead of allocating bandwidth collect buffers (CBs)
as a fallback for latency CBs, and spamming the kernel log
with failure messages, instead multiplex use across
the actual allocated number of latency CBs and just report
the failure to allocate once, with values to improve debugging.

Improves behaviour for scenarios where available CB resources
are spread across many VFs but VF drivers still see a lot
of (virtual) CPUs, which will easily be the case with the
default VF settings for Xen dom0.

Also, the low latency property is most critical for req.notify PQP
requests. Use high bandwidth CBs also for PQP operations other than
the REARM request, which is the performance critical req. for
req_notify_cq. This should improve performance for event based
applications under high load.

Orabug: 24424521

Signed-off-by: Knut Omang <knut.omang@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_dev.h b/drivers/infiniband/hw/sif/sif_dev.h
index 01e848be9a86d0213eca7d0b1f93efb9fc5945d3..1d349599cf883409010f5d7a01d3ef5bd33dc560 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_dev.h
+++ b/drivers/infiniband/hw/sif/sif_dev.h
@@ -281,7 +281,7 @@ struct sif_dev {
        struct sif_cb **kernel_cb[2]; /* cb's for the kernel (bw and low latency per cpu) */
        int pqp_cnt;              /* Number of PQPs set up */
        atomic_t next_pqp;        /* Used for round robin assignment of pqp */
-       int kernel_cb_cnt;        /* Number of pairs of CBs set up for the kernel */
+       int kernel_cb_cnt[2];     /* Number of CBs set up for the kernel for each kind */
        struct sif_idr xrcd_refs; /* Mgmt of sif_xrcd allocations */
        struct sif_idr pd_refs;   /* Mgmt of sif_pd allocations */
        struct sif_spqp_pool ki_spqp; /* Stencil PQPs for key invalidates */

diff --git a/drivers/infiniband/hw/sif/sif_hwi.c b/drivers/infiniband/hw/sif/sif_hwi.c
index 08f536ec05218f85a8025a5dff208ae2e7519a27..55fe9b76698292184c9a77a9cec280b7e8fd0e31 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_hwi.c
+++ b/drivers/infiniband/hw/sif/sif_hwi.c
@@ -177,13 +177,14 @@ static void sif_hw_kernel_cb_fini(struct sif_dev *sdev)
 {
        int i;
 
-       while (sdev->kernel_cb_cnt > 0) {
-               int j = sdev->kernel_cb_cnt - 1;
+       for (i = 0; i < 2; i++) {
+               while (sdev->kernel_cb_cnt[i] > 0) {
+                       int j = sdev->kernel_cb_cnt[i] - 1;
 
-               for (i = 0; i < 2; i++)
                        if (sdev->kernel_cb[i][j])
                                release_cb(sdev, sdev->kernel_cb[i][j]);
-               sdev->kernel_cb_cnt--;
+                       sdev->kernel_cb_cnt[i]--;
+               }
        }
        for (i = 0; i < 2; i++)
                kfree(sdev->kernel_cb[i]);
@@ -193,7 +194,7 @@ static void sif_hw_kernel_cb_fini(struct sif_dev *sdev)
 
 static int sif_hw_kernel_cb_init(struct sif_dev *sdev)
 {
-       int i;
+       int i, j;
        uint n_cbs = min(sif_cb_max, num_present_cpus());
 
        if (!n_cbs)
@@ -205,19 +206,25 @@ static int sif_hw_kernel_cb_init(struct sif_dev *sdev)
                        goto alloc_failed;
        }
 
-       for (i = 0; i < n_cbs; i++) {
-               sdev->kernel_cb[0][i] = alloc_cb(sdev, false);
-               if (!sdev->kernel_cb[0][i])
-                       goto alloc_failed;
-               sdev->kernel_cb[1][i] = alloc_cb(sdev, true);
-               if (!sdev->kernel_cb[1][i])
+       for (i = 0; i < 2; i++) {
+               for (j = 0; j < n_cbs; j++) {
+                       sdev->kernel_cb[i][j] = alloc_cb(sdev, i);
+                       if (!sdev->kernel_cb[i][j]) {
+                               sif_log(sdev, SIF_INFO,
+                                       "Failed to allocate more than %d of %d requested %s CBs",
+                                       j, n_cbs,
+                                       (QOSL_LOW_LATENCY ? "low latency" : "high bandwidth"));
+                               break;
+                       }
+               }
+               /* Require at least one collect buffer of each kind */
+               if (j == 0)
                        goto alloc_failed;
+               sdev->kernel_cb_cnt[i] = j;
        }
-       sdev->kernel_cb_cnt = i;
        return 0;
 
 alloc_failed:
-       sdev->kernel_cb_cnt = i;
        sif_hw_kernel_cb_fini(sdev);
        return -ENOMEM;
 }

diff --git a/drivers/infiniband/hw/sif/sif_pd.c b/drivers/infiniband/hw/sif/sif_pd.c
index 778d0c2856cca26a185d3cbf877a9dda86116b4d..b947efec58379dca0a41aaf8878edc903b9f85a4 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pd.c
+++ b/drivers/infiniband/hw/sif/sif_pd.c
@@ -252,14 +252,10 @@ struct sif_cb *alloc_cb(struct sif_dev *sdev, bool lat_cb)
 
        if (unlikely(lat_cb)) {
                idx = sif_alloc_lat_cb_idx(sdev);
-               if (idx < 0) {
-                       sif_log(sdev, SIF_INFO, "Unable to allocate lat_cb - trying bw_cb instead");
-                       lat_cb = false;
-               } else
-                       cb->cb = get_lat_cb(sdev, idx);
-       }
-
-       if (likely(!lat_cb)) {
+               if (idx < 0)
+                       goto err_index;
+               cb->cb = get_lat_cb(sdev, idx);
+       } else {
                idx = sif_alloc_bw_cb_idx(sdev);
                if (idx < 0)
                        goto err_index;
@@ -308,7 +304,7 @@ struct sif_cb *sif_cb_from_uc(struct sif_ucontext *uc, u32 index)
 int sif_cb_write(struct sif_qp *qp, struct psif_wr *wqe, int cp_len)
 {
        unsigned long flags;
-       struct sif_cb *cb = get_cb(qp);
+       struct sif_cb *cb = get_cb(qp, wqe);
 
        if (!spin_trylock_irqsave(&cb->lock, flags))
                return -EBUSY;
@@ -332,7 +328,7 @@ void sif_doorbell_write(struct sif_qp *qp, struct psif_wr *wqe, bool start)
 {
        unsigned long flags;
        u16 doorbell_offset = start ? SQS_START_DOORBELL : SQS_STOP_DOORBELL;
-       struct sif_cb *cb = get_cb(qp);
+       struct sif_cb *cb = get_cb(qp, wqe);
        struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device);
 
        sif_log(sdev, SIF_QP, "%s sqs for qp %d sq_seq %d", (start ? "start" : "stop"),
@@ -354,10 +350,10 @@ void sif_doorbell_write(struct sif_qp *qp, struct psif_wr *wqe, bool start)
 void sif_doorbell_from_sqe(struct sif_qp *qp, u16 seq, bool start)
 {
        u16 doorbell_offset = start ? SQS_START_DOORBELL : SQS_STOP_DOORBELL;
-       struct sif_cb *cb = get_cb(qp);
        struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device);
        struct sif_sq *sq = get_sif_sq(sdev, qp->qp_idx);
        u64 *wqe = (u64 *)get_sq_entry(sq, seq);
+       struct sif_cb *cb = get_cb(qp, (struct psif_wr *)wqe);
 
        /* Pick the 1st 8 bytes directly from the sq entry: */
        wmb();

diff --git a/drivers/infiniband/hw/sif/sif_pqp.c b/drivers/infiniband/hw/sif/sif_pqp.c
index 6b9228d92bee9041157fa266a3ef2b8c5ac5e58c..94fbe83bd8a77a2eb3898c153ac8c4241f2503d2 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pqp.c
+++ b/drivers/infiniband/hw/sif/sif_pqp.c
@@ -423,13 +423,17 @@ struct sif_pqp *get_next_pqp(struct sif_dev *sdev)
        return pqp;
 }
 
-struct sif_cb *get_cb(struct sif_qp *qp)
+struct sif_cb *get_cb(struct sif_qp *qp, struct psif_wr *wr)
 {
        struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device);
        unsigned int cpu = smp_processor_id();
-       return sdev->kernel_cb[qp->qosl][cpu % sdev->kernel_cb_cnt];
-}
+       enum psif_tsu_qos cb_type = qp->qosl;
+       /* Only use low latency CBs for the frequently occuring notify events (REARM) */
+       if (cb_type == QOSL_LOW_LATENCY && wr->op != PSIF_WR_REARM_CMPL_EVENT)
+               cb_type = QOSL_HIGH_BANDWIDTH;
 
+       return sdev->kernel_cb[cb_type][cpu % sdev->kernel_cb_cnt[cb_type]];
+}
 
 inline bool pqp_req_gets_completion(struct sif_pqp *pqp, struct psif_wr *wr, enum post_mode mode)
 {

diff --git a/drivers/infiniband/hw/sif/sif_pqp.h b/drivers/infiniband/hw/sif/sif_pqp.h
index 55bcd7ce3880986be1463733a39237de01dcaad5..106aec75e2585a03bf4bdc2345b44e12c6231076 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pqp.h
+++ b/drivers/infiniband/hw/sif/sif_pqp.h
@@ -95,8 +95,8 @@ struct sif_pqp *get_pqp_same_eq(struct sif_dev *sdev, int comp_vector);
 /* Get the next PQP in round robin fashion */
 struct sif_pqp *get_next_pqp(struct sif_dev *sdev);
 
-/* Get the right CB for the current CPU for the given QP */
-struct sif_cb *get_cb(struct sif_qp *qp);
+/* Get the right CB for the current CPU for the given QP and wr */
+struct sif_cb *get_cb(struct sif_qp *qp, struct psif_wr *wr);
 
 static inline struct sif_cq *pqp_cq(struct sif_dev *sdev)
 {