From 2bbf6158c7698de9248107af97df2039e2f4f6af Mon Sep 17 00:00:00 2001 From: Wei Lin Guay Date: Tue, 30 May 2017 16:11:29 +0200 Subject: [PATCH] net/rds: remove the RDS specific path record caching MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This patch partially reverts commit b12826152417 ("RDS: SA query optimization"), which has RDS specific path record caching, and uses the underlying ibacm path record caching. ibacm considers all entries as a similar path record (N is the TOS). Thus, RDS needs to update the SL manually during the QP creation. RDS also assumes that it is a 1:1 mapping in the TOS to SL mapping. Orabug: 26124147 Signed-off-by: Wei Lin Guay Signed-off-by: Santosh Shilimkar Reviewed-by: Avinash Repaka Reviewed-by: HÃ¥kon Bugge --- net/rds/connection.c | 39 ------------------------------ net/rds/rdma_transport.c | 51 ++++++++-------------------------------- net/rds/rds.h | 2 -- 3 files changed, 10 insertions(+), 82 deletions(-) diff --git a/net/rds/connection.c b/net/rds/connection.c index 80ce8baa83b1c..af7c14dfb0142 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -774,7 +774,6 @@ static char *conn_drop_reasons[] = { [DR_INV_CONN_STATE] = "invalid connection state", [DR_DOWN_TRANSITION_FAIL] = "failure to move to DOWN state", [DR_CONN_DESTROY] = "connection destroy", - [DR_ZERO_LANE_DOWN] = "zero lane went down", [DR_CONN_CONNECT_FAIL] = "conn_connect failure", [DR_HB_TIMEOUT] = "hb timeout", [DR_RECONNECT_TIMEOUT] = "reconnect timeout", @@ -789,7 +788,6 @@ static char *conn_drop_reasons[] = { [DR_IB_RDMA_ACCEPT_FAIL] = "rdma_accept failure", [DR_IB_ACT_SETUP_QP_FAIL] = "active setup_qp failure", [DR_IB_RDMA_CONNECT_FAIL] = "rdma_connect failure", - [DR_IB_SET_IB_PATH_FAIL] = "rdma_set_ib_paths failure", [DR_IB_RESOLVE_ROUTE_FAIL] = "resolve_route failure", [DR_IB_RDMA_CM_ID_MISMATCH] = "detected rdma_cm_id mismatch", [DR_IB_ROUTE_ERR] = "ROUTE_ERROR event", @@ -835,37 +833,6 @@ char *conn_drop_reason_str(enum rds_conn_drop_src reason) ARRAY_SIZE(conn_drop_reasons), reason); } -static void rds_conn_probe_lanes(struct rds_connection *conn) -{ - struct hlist_head *head = - rds_conn_bucket(conn->c_laddr, conn->c_faddr); - struct rds_connection *tmp; - - /* XXX only do this for IB transport? */ - rcu_read_lock(); - hlist_for_each_entry_rcu(tmp, head, c_hash_node) { - if (tmp->c_faddr == conn->c_faddr && - tmp->c_laddr == conn->c_laddr && - tmp->c_tos != 0 && - tmp->c_trans == conn->c_trans) { - if (rds_conn_up(tmp)) - rds_send_hb(tmp, 0); - else if (rds_conn_connecting(tmp) && - (tmp->c_path[0].cp_route_resolved == 0)) { - printk(KERN_INFO "RDS/IB: connection " - "<%u.%u.%u.%u,%u.%u.%u.%u,%d> " - "connecting, force reset ", - NIPQUAD(tmp->c_laddr), - NIPQUAD(tmp->c_faddr), - tmp->c_tos); - - rds_conn_drop(tmp, DR_ZERO_LANE_DOWN); - } - } - } - rcu_read_unlock(); -} - /* * Force a disconnect */ @@ -890,9 +857,6 @@ void rds_conn_path_drop(struct rds_conn_path *cp, int reason) conn->c_tos, conn_drop_reason_str(cp->cp_drop_source)); - if (conn->c_tos == 0) - rds_conn_probe_lanes(conn); - } else if ((cp->cp_reconnect_warn) && (now - cp->cp_reconnect_start > 60)) { printk(KERN_INFO "RDS/%s: re-connect " @@ -905,9 +869,6 @@ void rds_conn_path_drop(struct rds_conn_path *cp, int reason) cp->cp_reconnect_drops, cp->cp_reconnect_err); cp->cp_reconnect_warn = 0; - - if (conn->c_tos == 0) - rds_conn_probe_lanes(conn); } cp->cp_reconnect_drops++; diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c index 4dfd784a83551..6c1578ed97188 100644 --- a/net/rds/rdma_transport.c +++ b/net/rds/rdma_transport.c @@ -103,7 +103,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, * while we're executing. */ if (conn) { mutex_lock(&conn->c_cm_lock); - /* If the connection is being shut down, bail out * right away. We return 0 so cm_id doesn't get * destroyed prematurely */ @@ -124,44 +123,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_ADDR_RESOLVED: rdma_set_service_type(cm_id, conn->c_tos); - if (conn->c_tos && conn->c_reconnect) { - struct rds_ib_connection *base_ic = - conn->c_base_conn->c_transport_data; - - mutex_lock(&conn->c_base_conn->c_cm_lock); - if (rds_conn_transition(conn->c_base_conn, RDS_CONN_UP, - RDS_CONN_UP)) { - ret = rdma_set_ib_paths(cm_id, - base_ic->i_cm_id->route.path_rec, - base_ic->i_cm_id->route.num_paths); - if (!ret) { - struct rds_ib_connection *ic = - conn->c_transport_data; - - cm_id->route.path_rec[0].sl = - ic->i_sl; - cm_id->route.path_rec[0].qos_class = - conn->c_tos; - ret = trans->cm_initiate_connect(cm_id); - } - } else { - ret = 1; - } - mutex_unlock(&conn->c_base_conn->c_cm_lock); - - if (ret) { - rds_rtd(RDS_RTD_CM, - "ADDR_RESOLVED: ret %d, calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n", - ret, NIPQUAD(conn->c_laddr), - NIPQUAD(conn->c_faddr), conn->c_tos); - rds_conn_drop(conn, DR_IB_SET_IB_PATH_FAIL); - ret = 0; - } - - break; - } - - /* XXX do we need to clean up if this fails? */ ret = rdma_resolve_route(cm_id, rds_rdma_resolve_to_ms[conn->c_to_index]); @@ -194,9 +155,17 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, struct rds_ib_connection *ibic; ibic = conn->c_transport_data; - if (ibic && ibic->i_cm_id == cm_id) + if (ibic && ibic->i_cm_id == cm_id) { + /* ibacm caches the path record without considering the tos/sl. + * It is considered a match if the matches the + * cache. In order to create qp with the correct sl/vl, RDS + * needs to update the sl manually. As for now, RDS is assuming + * that it is a 1:1 in tos to sl mapping. + */ + cm_id->route.path_rec[0].sl = conn->c_tos; + cm_id->route.path_rec[0].qos_class = conn->c_tos; ret = trans->cm_initiate_connect(cm_id); - else { + } else { rds_rtd(RDS_RTD_CM, "ROUTE_RESOLVED: calling rds_conn_drop, conn %p <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n", conn, NIPQUAD(conn->c_laddr), diff --git a/net/rds/rds.h b/net/rds/rds.h index 0b74ca9934bb7..8aef3237442ab 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -154,7 +154,6 @@ enum rds_conn_drop_src { DR_INV_CONN_STATE, DR_DOWN_TRANSITION_FAIL, DR_CONN_DESTROY, - DR_ZERO_LANE_DOWN, DR_CONN_CONNECT_FAIL, DR_HB_TIMEOUT, DR_RECONNECT_TIMEOUT, @@ -173,7 +172,6 @@ enum rds_conn_drop_src { DR_IB_RDMA_CONNECT_FAIL, /* event handling */ - DR_IB_SET_IB_PATH_FAIL, DR_IB_RESOLVE_ROUTE_FAIL, DR_IB_RDMA_CM_ID_MISMATCH, DR_IB_ROUTE_ERR, -- 2.50.1