RDS: IB: restore the vector spreading for the CQs

Since the IB_CQ_LEAST_LOADED vector support is not their on newer
kernels(post OFED 1.5), we had #if 0 code for it which got removed
as part of 'commit 3f1db626594e ("RDS: IB: drop discontinued IB
CQ_VECTOR support")'. On UEK2, the drivers had implementation
for this IB verb. UEK4 which is based on newer kernel obviously
doesn't support it.

RDS had an alternate fallback scheme which can be used in absence
of the dropped verb. On UEK2, we didn't use it but UEK4 RDS code was
silently using that till the code got removed. The patch restores
that code with bit more clarity on what it is actually doing.

Orabug: 23550561

Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 49683f752837e5f9c936fda0b09764251ba66324..3dc0f9886dbdf27d3667e057037b751d455c973b 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -620,6 +620,28 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
        }
 }
 
+static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
+{
+       int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1];
+       int index = rds_ibdev->dev->num_comp_vectors - 1;
+       int i;
+
+       for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) {
+               if (rds_ibdev->vector_load[i] < min) {
+                       index = i;
+                       min = rds_ibdev->vector_load[i];
+               }
+       }
+
+       rds_ibdev->vector_load[index]++;
+       return index;
+}
+
+static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index)
+{
+       rds_ibdev->vector_load[index]--;
+}
+
 /*
  * This needs to be very careful to not leave IS_ERR pointers around for
  * cleanup to trip over.
@@ -652,27 +674,34 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
        ic->i_pd = rds_ibdev->pd;
        ic->i_mr = rds_ibdev->mr;
 
+       ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
        ic->i_scq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
                                rds_ib_cq_event_handler, conn,
-                               ic->i_send_ring.w_nr + 1, 0);
+                               ic->i_send_ring.w_nr + 1,
+                               ic->i_scq_vector);
        if (IS_ERR(ic->i_scq)) {
                ret = PTR_ERR(ic->i_scq);
                ic->i_scq = NULL;
+               ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
                rdsdebug("ib_create_cq send failed: %d\n", ret);
                goto out;
        }
 
+       ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
        if (rds_ib_srq_enabled)
                ic->i_rcq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
                                        rds_ib_cq_event_handler, conn,
-                                       rds_ib_srq_max_wr - 1, 0);
+                                       rds_ib_srq_max_wr - 1,
+                                       ic->i_rcq_vector);
        else
                ic->i_rcq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
                                        rds_ib_cq_event_handler, conn,
-                                       ic->i_recv_ring.w_nr, 0);
+                                       ic->i_recv_ring.w_nr,
+                                       ic->i_rcq_vector);
        if (IS_ERR(ic->i_rcq)) {
                ret = PTR_ERR(ic->i_rcq);
                ic->i_rcq = NULL;
+               ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
                rdsdebug("ib_create_cq recv failed: %d\n", ret);
                goto out;
        }
@@ -1244,11 +1273,17 @@ void rds_ib_conn_shutdown(struct rds_connection *conn)
                if (ic->i_cm_id->qp)
                        rdma_destroy_qp(ic->i_cm_id);
 
-               if (ic->i_rcq)
+               if (ic->i_rcq) {
+                       if (ic->rds_ibdev)
+                               ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);
                        ib_destroy_cq(ic->i_rcq);
+               }
 
-               if (ic->i_scq)
+               if (ic->i_scq) {
+                       if (ic->rds_ibdev)
+                               ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);
                        ib_destroy_cq(ic->i_scq);
+               }
 
                /* then free the resources that ib callbacks use */
                if (ic->i_send_hdrs)