From 976977affa81716bad5d741add1a3d3d6b3a05e7 Mon Sep 17 00:00:00 2001 From: Knut Omang Date: Tue, 9 Aug 2016 11:07:23 +0200 Subject: [PATCH] sif: cb: Improve algorithm for allocating and using CBs from driver MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Instead of allocating bandwidth collect buffers (CBs) as a fallback for latency CBs, and spamming the kernel log with failure messages, instead multiplex use across the actual allocated number of latency CBs and just report the failure to allocate once, with values to improve debugging. Improves behaviour for scenarios where available CB resources are spread across many VFs but VF drivers still see a lot of (virtual) CPUs, which will easily be the case with the default VF settings for Xen dom0. Also, the low latency property is most critical for req.notify PQP requests. Use high bandwidth CBs also for PQP operations other than the REARM request, which is the performance critical req. for req_notify_cq. This should improve performance for event based applications under high load. Orabug: 24424521 Signed-off-by: Knut Omang Reviewed-by: HÃ¥kon Bugge --- drivers/infiniband/hw/sif/sif_dev.h | 2 +- drivers/infiniband/hw/sif/sif_hwi.c | 33 +++++++++++++++++------------ drivers/infiniband/hw/sif/sif_pd.c | 18 ++++++---------- drivers/infiniband/hw/sif/sif_pqp.c | 10 ++++++--- drivers/infiniband/hw/sif/sif_pqp.h | 4 ++-- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/drivers/infiniband/hw/sif/sif_dev.h b/drivers/infiniband/hw/sif/sif_dev.h index 01e848be9a86..1d349599cf88 100644 --- a/drivers/infiniband/hw/sif/sif_dev.h +++ b/drivers/infiniband/hw/sif/sif_dev.h @@ -281,7 +281,7 @@ struct sif_dev { struct sif_cb **kernel_cb[2]; /* cb's for the kernel (bw and low latency per cpu) */ int pqp_cnt; /* Number of PQPs set up */ atomic_t next_pqp; /* Used for round robin assignment of pqp */ - int kernel_cb_cnt; /* Number of pairs of CBs set up for the kernel */ + int kernel_cb_cnt[2]; /* Number of CBs set up for the kernel for each kind */ struct sif_idr xrcd_refs; /* Mgmt of sif_xrcd allocations */ struct sif_idr pd_refs; /* Mgmt of sif_pd allocations */ struct sif_spqp_pool ki_spqp; /* Stencil PQPs for key invalidates */ diff --git a/drivers/infiniband/hw/sif/sif_hwi.c b/drivers/infiniband/hw/sif/sif_hwi.c index 08f536ec0521..55fe9b766982 100644 --- a/drivers/infiniband/hw/sif/sif_hwi.c +++ b/drivers/infiniband/hw/sif/sif_hwi.c @@ -177,13 +177,14 @@ static void sif_hw_kernel_cb_fini(struct sif_dev *sdev) { int i; - while (sdev->kernel_cb_cnt > 0) { - int j = sdev->kernel_cb_cnt - 1; + for (i = 0; i < 2; i++) { + while (sdev->kernel_cb_cnt[i] > 0) { + int j = sdev->kernel_cb_cnt[i] - 1; - for (i = 0; i < 2; i++) if (sdev->kernel_cb[i][j]) release_cb(sdev, sdev->kernel_cb[i][j]); - sdev->kernel_cb_cnt--; + sdev->kernel_cb_cnt[i]--; + } } for (i = 0; i < 2; i++) kfree(sdev->kernel_cb[i]); @@ -193,7 +194,7 @@ static void sif_hw_kernel_cb_fini(struct sif_dev *sdev) static int sif_hw_kernel_cb_init(struct sif_dev *sdev) { - int i; + int i, j; uint n_cbs = min(sif_cb_max, num_present_cpus()); if (!n_cbs) @@ -205,19 +206,25 @@ static int sif_hw_kernel_cb_init(struct sif_dev *sdev) goto alloc_failed; } - for (i = 0; i < n_cbs; i++) { - sdev->kernel_cb[0][i] = alloc_cb(sdev, false); - if (!sdev->kernel_cb[0][i]) - goto alloc_failed; - sdev->kernel_cb[1][i] = alloc_cb(sdev, true); - if (!sdev->kernel_cb[1][i]) + for (i = 0; i < 2; i++) { + for (j = 0; j < n_cbs; j++) { + sdev->kernel_cb[i][j] = alloc_cb(sdev, i); + if (!sdev->kernel_cb[i][j]) { + sif_log(sdev, SIF_INFO, + "Failed to allocate more than %d of %d requested %s CBs", + j, n_cbs, + (QOSL_LOW_LATENCY ? "low latency" : "high bandwidth")); + break; + } + } + /* Require at least one collect buffer of each kind */ + if (j == 0) goto alloc_failed; + sdev->kernel_cb_cnt[i] = j; } - sdev->kernel_cb_cnt = i; return 0; alloc_failed: - sdev->kernel_cb_cnt = i; sif_hw_kernel_cb_fini(sdev); return -ENOMEM; } diff --git a/drivers/infiniband/hw/sif/sif_pd.c b/drivers/infiniband/hw/sif/sif_pd.c index 778d0c2856cc..b947efec5837 100644 --- a/drivers/infiniband/hw/sif/sif_pd.c +++ b/drivers/infiniband/hw/sif/sif_pd.c @@ -252,14 +252,10 @@ struct sif_cb *alloc_cb(struct sif_dev *sdev, bool lat_cb) if (unlikely(lat_cb)) { idx = sif_alloc_lat_cb_idx(sdev); - if (idx < 0) { - sif_log(sdev, SIF_INFO, "Unable to allocate lat_cb - trying bw_cb instead"); - lat_cb = false; - } else - cb->cb = get_lat_cb(sdev, idx); - } - - if (likely(!lat_cb)) { + if (idx < 0) + goto err_index; + cb->cb = get_lat_cb(sdev, idx); + } else { idx = sif_alloc_bw_cb_idx(sdev); if (idx < 0) goto err_index; @@ -308,7 +304,7 @@ struct sif_cb *sif_cb_from_uc(struct sif_ucontext *uc, u32 index) int sif_cb_write(struct sif_qp *qp, struct psif_wr *wqe, int cp_len) { unsigned long flags; - struct sif_cb *cb = get_cb(qp); + struct sif_cb *cb = get_cb(qp, wqe); if (!spin_trylock_irqsave(&cb->lock, flags)) return -EBUSY; @@ -332,7 +328,7 @@ void sif_doorbell_write(struct sif_qp *qp, struct psif_wr *wqe, bool start) { unsigned long flags; u16 doorbell_offset = start ? SQS_START_DOORBELL : SQS_STOP_DOORBELL; - struct sif_cb *cb = get_cb(qp); + struct sif_cb *cb = get_cb(qp, wqe); struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device); sif_log(sdev, SIF_QP, "%s sqs for qp %d sq_seq %d", (start ? "start" : "stop"), @@ -354,10 +350,10 @@ void sif_doorbell_write(struct sif_qp *qp, struct psif_wr *wqe, bool start) void sif_doorbell_from_sqe(struct sif_qp *qp, u16 seq, bool start) { u16 doorbell_offset = start ? SQS_START_DOORBELL : SQS_STOP_DOORBELL; - struct sif_cb *cb = get_cb(qp); struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device); struct sif_sq *sq = get_sif_sq(sdev, qp->qp_idx); u64 *wqe = (u64 *)get_sq_entry(sq, seq); + struct sif_cb *cb = get_cb(qp, (struct psif_wr *)wqe); /* Pick the 1st 8 bytes directly from the sq entry: */ wmb(); diff --git a/drivers/infiniband/hw/sif/sif_pqp.c b/drivers/infiniband/hw/sif/sif_pqp.c index 6b9228d92bee..94fbe83bd8a7 100644 --- a/drivers/infiniband/hw/sif/sif_pqp.c +++ b/drivers/infiniband/hw/sif/sif_pqp.c @@ -423,13 +423,17 @@ struct sif_pqp *get_next_pqp(struct sif_dev *sdev) return pqp; } -struct sif_cb *get_cb(struct sif_qp *qp) +struct sif_cb *get_cb(struct sif_qp *qp, struct psif_wr *wr) { struct sif_dev *sdev = to_sdev(qp->ibqp.pd->device); unsigned int cpu = smp_processor_id(); - return sdev->kernel_cb[qp->qosl][cpu % sdev->kernel_cb_cnt]; -} + enum psif_tsu_qos cb_type = qp->qosl; + /* Only use low latency CBs for the frequently occuring notify events (REARM) */ + if (cb_type == QOSL_LOW_LATENCY && wr->op != PSIF_WR_REARM_CMPL_EVENT) + cb_type = QOSL_HIGH_BANDWIDTH; + return sdev->kernel_cb[cb_type][cpu % sdev->kernel_cb_cnt[cb_type]]; +} inline bool pqp_req_gets_completion(struct sif_pqp *pqp, struct psif_wr *wr, enum post_mode mode) { diff --git a/drivers/infiniband/hw/sif/sif_pqp.h b/drivers/infiniband/hw/sif/sif_pqp.h index 55bcd7ce3880..106aec75e258 100644 --- a/drivers/infiniband/hw/sif/sif_pqp.h +++ b/drivers/infiniband/hw/sif/sif_pqp.h @@ -95,8 +95,8 @@ struct sif_pqp *get_pqp_same_eq(struct sif_dev *sdev, int comp_vector); /* Get the next PQP in round robin fashion */ struct sif_pqp *get_next_pqp(struct sif_dev *sdev); -/* Get the right CB for the current CPU for the given QP */ -struct sif_cb *get_cb(struct sif_qp *qp); +/* Get the right CB for the current CPU for the given QP and wr */ +struct sif_cb *get_cb(struct sif_qp *qp, struct psif_wr *wr); static inline struct sif_cq *pqp_cq(struct sif_dev *sdev) { -- 2.50.1