From 12d79d5f8552a7a6164e650dc58a9ca08c7a10b9 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Thu, 9 Jun 2016 21:05:54 -0700 Subject: [PATCH] RDS: IB: restore the vector spreading for the CQs Since the IB_CQ_LEAST_LOADED vector support is not their on newer kernels(post OFED 1.5), we had #if 0 code for it which got removed as part of 'commit 3f1db626594e ("RDS: IB: drop discontinued IB CQ_VECTOR support")'. On UEK2, the drivers had implementation for this IB verb. UEK4 which is based on newer kernel obviously doesn't support it. RDS had an alternate fallback scheme which can be used in absence of the dropped verb. On UEK2, we didn't use it but UEK4 RDS code was silently using that till the code got removed. The patch restores that code with bit more clarity on what it is actually doing. Orabug: 23550561 Signed-off-by: Santosh Shilimkar --- net/rds/ib_cm.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 49683f752837..3dc0f9886dbd 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -620,6 +620,28 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data) } } +static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev) +{ + int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1]; + int index = rds_ibdev->dev->num_comp_vectors - 1; + int i; + + for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) { + if (rds_ibdev->vector_load[i] < min) { + index = i; + min = rds_ibdev->vector_load[i]; + } + } + + rds_ibdev->vector_load[index]++; + return index; +} + +static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index) +{ + rds_ibdev->vector_load[index]--; +} + /* * This needs to be very careful to not leave IS_ERR pointers around for * cleanup to trip over. @@ -652,27 +674,34 @@ static int rds_ib_setup_qp(struct rds_connection *conn) ic->i_pd = rds_ibdev->pd; ic->i_mr = rds_ibdev->mr; + ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev); ic->i_scq = ib_create_cq(dev, rds_ib_cq_comp_handler_send, rds_ib_cq_event_handler, conn, - ic->i_send_ring.w_nr + 1, 0); + ic->i_send_ring.w_nr + 1, + ic->i_scq_vector); if (IS_ERR(ic->i_scq)) { ret = PTR_ERR(ic->i_scq); ic->i_scq = NULL; + ibdev_put_vector(rds_ibdev, ic->i_scq_vector); rdsdebug("ib_create_cq send failed: %d\n", ret); goto out; } + ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev); if (rds_ib_srq_enabled) ic->i_rcq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, rds_ib_cq_event_handler, conn, - rds_ib_srq_max_wr - 1, 0); + rds_ib_srq_max_wr - 1, + ic->i_rcq_vector); else ic->i_rcq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, rds_ib_cq_event_handler, conn, - ic->i_recv_ring.w_nr, 0); + ic->i_recv_ring.w_nr, + ic->i_rcq_vector); if (IS_ERR(ic->i_rcq)) { ret = PTR_ERR(ic->i_rcq); ic->i_rcq = NULL; + ibdev_put_vector(rds_ibdev, ic->i_rcq_vector); rdsdebug("ib_create_cq recv failed: %d\n", ret); goto out; } @@ -1244,11 +1273,17 @@ void rds_ib_conn_shutdown(struct rds_connection *conn) if (ic->i_cm_id->qp) rdma_destroy_qp(ic->i_cm_id); - if (ic->i_rcq) + if (ic->i_rcq) { + if (ic->rds_ibdev) + ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector); ib_destroy_cq(ic->i_rcq); + } - if (ic->i_scq) + if (ic->i_scq) { + if (ic->rds_ibdev) + ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector); ib_destroy_cq(ic->i_scq); + } /* then free the resources that ib callbacks use */ if (ic->i_send_hdrs) -- 2.50.1