net/rds: remove the RDS specific path record caching

This patch partially reverts commit b12826152417 ("RDS: SA query
optimization"), which has RDS specific path record caching, and uses the
underlying ibacm path record caching. ibacm considers all <source,dest,N>
entries as a similar path record (N is the TOS). Thus, RDS needs to update
the SL manually during the QP creation. RDS also assumes that it is a 1:1
mapping in the TOS to SL mapping.

Orabug: 26124147

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>
Reviewed-by: Avinash Repaka <avinash.repaka@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

diff --git a/net/rds/connection.c b/net/rds/connection.c
index 80ce8baa83b1c7de27137cf91e7845b8f051ee91..af7c14dfb01423a23753c42eaed75b5579a6b702 100644 (file)
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -774,7 +774,6 @@ static char *conn_drop_reasons[] = {
        [DR_INV_CONN_STATE]             = "invalid connection state",
        [DR_DOWN_TRANSITION_FAIL]       = "failure to move to DOWN state",
        [DR_CONN_DESTROY]               = "connection destroy",
-       [DR_ZERO_LANE_DOWN]             = "zero lane went down",
        [DR_CONN_CONNECT_FAIL]          = "conn_connect failure",
        [DR_HB_TIMEOUT]                 = "hb timeout",
        [DR_RECONNECT_TIMEOUT]          = "reconnect timeout",
@@ -789,7 +788,6 @@ static char *conn_drop_reasons[] = {
        [DR_IB_RDMA_ACCEPT_FAIL]        = "rdma_accept failure",
        [DR_IB_ACT_SETUP_QP_FAIL]       = "active setup_qp failure",
        [DR_IB_RDMA_CONNECT_FAIL]       = "rdma_connect failure",
-       [DR_IB_SET_IB_PATH_FAIL]        = "rdma_set_ib_paths failure",
        [DR_IB_RESOLVE_ROUTE_FAIL]      = "resolve_route failure",
        [DR_IB_RDMA_CM_ID_MISMATCH]     = "detected rdma_cm_id mismatch",
        [DR_IB_ROUTE_ERR]               = "ROUTE_ERROR event",
@@ -835,37 +833,6 @@ char *conn_drop_reason_str(enum rds_conn_drop_src reason)
                             ARRAY_SIZE(conn_drop_reasons), reason);
 }
 
-static void rds_conn_probe_lanes(struct rds_connection *conn)
-{
-       struct hlist_head *head =
-               rds_conn_bucket(conn->c_laddr, conn->c_faddr);
-       struct rds_connection *tmp;
-
-       /* XXX only do this for IB transport? */
-       rcu_read_lock();
-       hlist_for_each_entry_rcu(tmp, head, c_hash_node) {
-               if (tmp->c_faddr == conn->c_faddr &&
-                       tmp->c_laddr == conn->c_laddr &&
-                       tmp->c_tos != 0 &&
-                       tmp->c_trans == conn->c_trans) {
-                       if (rds_conn_up(tmp))
-                               rds_send_hb(tmp, 0);
-                       else if (rds_conn_connecting(tmp) &&
-                                (tmp->c_path[0].cp_route_resolved == 0)) {
-                               printk(KERN_INFO "RDS/IB: connection "
-                                      "<%u.%u.%u.%u,%u.%u.%u.%u,%d> "
-                                      "connecting, force reset ",
-                                      NIPQUAD(tmp->c_laddr),
-                                      NIPQUAD(tmp->c_faddr),
-                                      tmp->c_tos);
-
-                               rds_conn_drop(tmp, DR_ZERO_LANE_DOWN);
-                       }
-               }
-       }
-       rcu_read_unlock();
-}
-
 /*
  * Force a disconnect
  */
@@ -890,9 +857,6 @@ void rds_conn_path_drop(struct rds_conn_path *cp, int reason)
                                conn->c_tos,
                                conn_drop_reason_str(cp->cp_drop_source));
 
-               if (conn->c_tos == 0)
-                       rds_conn_probe_lanes(conn);
-
        } else if ((cp->cp_reconnect_warn) &&
                   (now - cp->cp_reconnect_start > 60)) {
                printk(KERN_INFO "RDS/%s: re-connect "
@@ -905,9 +869,6 @@ void rds_conn_path_drop(struct rds_conn_path *cp, int reason)
                        cp->cp_reconnect_drops,
                        cp->cp_reconnect_err);
                cp->cp_reconnect_warn = 0;
-
-               if (conn->c_tos == 0)
-                       rds_conn_probe_lanes(conn);
        }
        cp->cp_reconnect_drops++;
 

diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index 4dfd784a835516170e0903a2d9bb057d19475902..6c1578ed9718859945f8af3ca07f6a705666602f 100644 (file)
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -103,7 +103,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
         * while we're executing. */
        if (conn) {
                mutex_lock(&conn->c_cm_lock);
-
                /* If the connection is being shut down, bail out
                 * right away. We return 0 so cm_id doesn't get
                 * destroyed prematurely */
@@ -124,44 +123,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
        case RDMA_CM_EVENT_ADDR_RESOLVED:
                rdma_set_service_type(cm_id, conn->c_tos);
 
-               if (conn->c_tos && conn->c_reconnect) {
-                       struct rds_ib_connection *base_ic =
-                               conn->c_base_conn->c_transport_data;
-
-                       mutex_lock(&conn->c_base_conn->c_cm_lock);
-                       if (rds_conn_transition(conn->c_base_conn, RDS_CONN_UP,
-                                               RDS_CONN_UP)) {
-                               ret = rdma_set_ib_paths(cm_id,
-                                       base_ic->i_cm_id->route.path_rec,
-                                       base_ic->i_cm_id->route.num_paths);
-                               if (!ret) {
-                                       struct rds_ib_connection *ic =
-                                               conn->c_transport_data;
-
-                                       cm_id->route.path_rec[0].sl =
-                                               ic->i_sl;
-                                       cm_id->route.path_rec[0].qos_class =
-                                               conn->c_tos;
-                                       ret = trans->cm_initiate_connect(cm_id);
-                               }
-                       } else {
-                               ret = 1;
-                       }
-                       mutex_unlock(&conn->c_base_conn->c_cm_lock);
-
-                       if (ret) {
-                               rds_rtd(RDS_RTD_CM,
-                                       "ADDR_RESOLVED: ret %d, calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
-                                       ret, NIPQUAD(conn->c_laddr),
-                                       NIPQUAD(conn->c_faddr), conn->c_tos);
-                               rds_conn_drop(conn, DR_IB_SET_IB_PATH_FAIL);
-                               ret = 0;
-                       }
-
-                       break;
-               }
-
-
                /* XXX do we need to clean up if this fails? */
                ret = rdma_resolve_route(cm_id,
                                rds_rdma_resolve_to_ms[conn->c_to_index]);
@@ -194,9 +155,17 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
                        struct rds_ib_connection *ibic;
 
                        ibic = conn->c_transport_data;
-                       if (ibic && ibic->i_cm_id == cm_id)
+                       if (ibic && ibic->i_cm_id == cm_id) {
+                               /* ibacm caches the path record without considering the tos/sl.
+                                * It is considered a match if the <src,dest> matches the
+                                * cache. In order to create qp with the correct sl/vl, RDS
+                                * needs to update the sl manually. As for now, RDS is assuming
+                                * that it is a 1:1 in tos to sl mapping.
+                                */
+                               cm_id->route.path_rec[0].sl = conn->c_tos;
+                               cm_id->route.path_rec[0].qos_class = conn->c_tos;
                                ret = trans->cm_initiate_connect(cm_id);
-                       else {
+                       } else {
                                rds_rtd(RDS_RTD_CM,
                                        "ROUTE_RESOLVED: calling rds_conn_drop, conn %p <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
                                        conn, NIPQUAD(conn->c_laddr),

diff --git a/net/rds/rds.h b/net/rds/rds.h
index 0b74ca9934bb77511f09e64b41e405d26ddc2967..8aef3237442abc099fe1cf819023fa687ed68b06 100644 (file)
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -154,7 +154,6 @@ enum rds_conn_drop_src {
        DR_INV_CONN_STATE,
        DR_DOWN_TRANSITION_FAIL,
        DR_CONN_DESTROY,
-       DR_ZERO_LANE_DOWN,
        DR_CONN_CONNECT_FAIL,
        DR_HB_TIMEOUT,
        DR_RECONNECT_TIMEOUT,
@@ -173,7 +172,6 @@ enum rds_conn_drop_src {
        DR_IB_RDMA_CONNECT_FAIL,
 
        /* event handling */
-       DR_IB_SET_IB_PATH_FAIL,
        DR_IB_RESOLVE_ROUTE_FAIL,
        DR_IB_RDMA_CM_ID_MISMATCH,
        DR_IB_ROUTE_ERR,