From 5e86bae962373cc71c860fb9cb992aef9e039786 Mon Sep 17 00:00:00 2001 From: Bang Nguyen Date: Wed, 11 Sep 2013 14:09:04 -0700 Subject: [PATCH] RDMA CM: Add reason code for IB_CM_REJ_CONSUMER_DEFINED RDS: Support rolling downgrade from version 4.1 to 3.1 Orabug: 17484682 Signed-off-by: Giri Adari Signed-off-by: Richard Frank Signed-off-by: Chien-Hua Yen (cherry picked from commit 7b66ddd7f6a5b023191d74949fab41af245775a3) Signed-off-by: Jerry Snitselaar Conflicts: net/rds/rds.h (cherry picked from commit 0373566ba0d74f655ae83e09748f7cc8d553f351) --- drivers/infiniband/core/cm.c | 8 ++++++-- drivers/infiniband/core/cma.c | 4 ++-- net/rds/connection.c | 5 +++-- net/rds/ib.c | 10 +++++++--- net/rds/ib_cm.c | 23 +++++++---------------- net/rds/rdma_transport.c | 26 +++++++++++--------------- net/rds/rds.h | 8 ++++++-- net/rds/send.c | 30 +++++++++++++++++++++++++++++- net/rds/threads.c | 13 +++++++++++-- 9 files changed, 82 insertions(+), 45 deletions(-) diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 0271608a51c40..2831387a4eb72 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -876,8 +876,10 @@ retest: spin_unlock_irq(&cm_id_priv->lock); } else { spin_unlock_irq(&cm_id_priv->lock); + if (!err) + err = -EINVAL; ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, - NULL, 0, NULL, 0); + NULL, 0, &err, sizeof(err)); } break; case IB_CM_REP_SENT: @@ -888,8 +890,10 @@ retest: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: spin_unlock_irq(&cm_id_priv->lock); + if (!err) + err = -EINVAL; ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, - NULL, 0, NULL, 0); + NULL, 0, &err, sizeof(err)); break; case IB_CM_ESTABLISHED: spin_unlock_irq(&cm_id_priv->lock); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 38ffe09815035..3eef7e352a728 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -1102,7 +1102,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv) reject: cma_modify_qp_err(id_priv); ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, - NULL, 0, NULL, 0); + NULL, 0, &ret, sizeof(int)); return ret; } @@ -3049,7 +3049,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) return 0; reject: cma_modify_qp_err(id_priv); - rdma_reject(id, NULL, 0); + rdma_reject(id, &ret, sizeof(int)); return ret; } EXPORT_SYMBOL(rdma_accept); diff --git a/net/rds/connection.c b/net/rds/connection.c index 069ffcf1002b9..3c3283c3dd903 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -214,6 +214,7 @@ static struct rds_connection *__rds_conn_create(__be32 laddr, __be32 faddr, INIT_DELAYED_WORK(&conn->c_recv_w, rds_recv_worker); INIT_DELAYED_WORK(&conn->c_conn_w, rds_connect_worker); INIT_DELAYED_WORK(&conn->c_hb_w, rds_hb_worker); + INIT_DELAYED_WORK(&conn->c_reject_w, rds_reject_worker); INIT_WORK(&conn->c_down_w, rds_shutdown_worker); mutex_init(&conn->c_cm_lock); conn->c_flags = 0; @@ -293,7 +294,7 @@ struct rds_connection *rds_conn_find(__be32 laddr, __be32 faddr, } EXPORT_SYMBOL_GPL(rds_conn_find); -void rds_conn_shutdown(struct rds_connection *conn) +void rds_conn_shutdown(struct rds_connection *conn, int restart) { /* shut it down unless it's down already */ if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_DOWN)) { @@ -343,7 +344,7 @@ void rds_conn_shutdown(struct rds_connection *conn) * conn - the reconnect is always triggered by the active peer. */ cancel_delayed_work_sync(&conn->c_conn_w); rcu_read_lock(); - if (!hlist_unhashed(&conn->c_hash_node)) { + if (!hlist_unhashed(&conn->c_hash_node) && restart) { rcu_read_unlock(); rds_queue_reconnect(conn); } else { diff --git a/net/rds/ib.c b/net/rds/ib.c index 0acf30cc636f3..628430bdd83a1 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -295,7 +295,7 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn, void *buffer) { struct rds_info_rdma_connection *iinfo = buffer; - struct rds_ib_connection *ic; + struct rds_ib_connection *ic = conn->c_transport_data; /* We will only ever look at IB transports */ if (conn->c_trans != &rds_ib_transport) @@ -306,6 +306,12 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn, memset(&iinfo->src_gid, 0, sizeof(iinfo->src_gid)); memset(&iinfo->dst_gid, 0, sizeof(iinfo->dst_gid)); + + if (ic) { + iinfo->tos = conn->c_tos; + iinfo->sl = ic->i_sl; + } + if (rds_conn_state(conn) == RDS_CONN_UP) { struct rds_ib_device *rds_ibdev; struct rdma_dev_addr *dev_addr; @@ -332,8 +338,6 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn, iinfo->max_recv_wr = ic->i_recv_ring.w_nr; iinfo->max_send_sge = rds_ibdev->max_sge; rds_ib_get_mr_info(rds_ibdev, iinfo); - iinfo->tos = ic->conn->c_tos; - iinfo->sl = ic->i_sl; iinfo->cache_allocs = atomic_read(&ic->i_cache_allocs); } return 1; diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 083425602e0d4..2c97d639acb0e 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -732,20 +732,13 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, goto out; } - if (conn->c_reconnect && (conn->c_version > version)) { - printk(KERN_WARNING "RDS/IB: connection " - "<%u.%u.%u.%u,%u.%u.%u.%u,%d,%u.%u> rejecting version " - "(%u/%u)\n", - NIPQUAD(conn->c_laddr), - NIPQUAD(conn->c_faddr), - conn->c_tos, - RDS_PROTOCOL_MAJOR(conn->c_version), - RDS_PROTOCOL_MINOR(conn->c_version), - RDS_PROTOCOL_MAJOR(version), - RDS_PROTOCOL_MINOR(version)); - - conn = NULL; - goto out; + if (dp->dp_tos && !conn->c_base_conn) { + conn->c_base_conn = rds_conn_create(dp->dp_daddr, dp->dp_saddr, + &rds_ib_transport, 0, GFP_KERNEL); + if (IS_ERR(conn->c_base_conn)) { + conn = NULL; + goto out; + } } /* @@ -872,8 +865,6 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) } #endif - /* If the peer doesn't do protocol negotiation, we must - * default to RDSv3.0 */ rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1); ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c index a97c862aa54bd..9e7c4bcf89c00 100644 --- a/net/rds/rdma_transport.c +++ b/net/rds/rdma_transport.c @@ -186,23 +186,19 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_REJECTED: err = (int *)event->param.conn.private_data; if (conn) { - if (!conn->c_reconnect || conn->c_committed_version == - RDS_PROTOCOL_COMPAT_VERSION) { - if ((*err) == 0 && - event->status == RDS_REJ_CONSUMER_DEFINED) { - /* rejection from 3.x protocol */ - if (!conn->c_tos) { - /* retry the connect with a - lower compatible protocol */ - conn->c_proposed_version = - RDS_PROTOCOL_COMPAT_VERSION; - } - } else { + if (event->status == RDS_REJ_CONSUMER_DEFINED && (*err) == 0) { + /* Rejection from RDSV3.1 */ + if (!conn->c_tos) { conn->c_proposed_version = - RDS_PROTOCOL_VERSION; + RDS_PROTOCOL_COMPAT_VERSION; + rds_conn_drop(conn); + } else { + queue_delayed_work(rds_wq, + &conn->c_reject_w, + msecs_to_jiffies(10)); } - } - rds_conn_drop(conn); + } else + rds_conn_drop(conn); } break; diff --git a/net/rds/rds.h b/net/rds/rds.h index 7c1a06ea2b45a..4e747773f4495 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -128,6 +128,7 @@ struct rds_connection { struct delayed_work c_send_w; struct delayed_work c_recv_w; struct delayed_work c_conn_w; + struct delayed_work c_reject_w; struct delayed_work c_hb_w; struct work_struct c_down_w; struct mutex c_cm_lock; /* protect conn state & cm */ @@ -142,7 +143,6 @@ struct rds_connection { /* Protocol version */ unsigned int c_proposed_version; - unsigned int c_committed_version; unsigned int c_version; /* Re-connect stall diagnostics */ @@ -162,6 +162,8 @@ struct rds_connection { unsigned long c_hb_start; struct rds_connection *c_base_conn; + + unsigned int c_route_to_base; }; #define RDS_FLAG_CONG_BITMAP 0x01 @@ -670,7 +672,7 @@ struct rds_connection *rds_conn_create_outgoing(__be32 laddr, __be32 faddr, u8 tos, gfp_t gfp); struct rds_connection *rds_conn_find(__be32 laddr, __be32 faddr, struct rds_transport *trans, u8 tos); -void rds_conn_shutdown(struct rds_connection *conn); +void rds_conn_shutdown(struct rds_connection *conn, int restart); void rds_conn_destroy(struct rds_connection *conn); void rds_conn_reset(struct rds_connection *conn); void rds_conn_drop(struct rds_connection *conn); @@ -788,6 +790,7 @@ struct rds_message *rds_send_get_message(struct rds_connection *, struct rm_rdma_op *); int rds_send_internal(struct rds_connection *conn, struct rds_sock *rs, struct sk_buff *skb, gfp_t gfp); +void rds_route_to_base(struct rds_connection *conn); extern unsigned int rds_async_send_enabled; @@ -860,6 +863,7 @@ void rds_queue_reconnect(struct rds_connection *conn); void rds_connect_worker(struct work_struct *); void rds_shutdown_worker(struct work_struct *); void rds_send_worker(struct work_struct *); +void rds_reject_worker(struct work_struct *); void rds_recv_worker(struct work_struct *); void rds_hb_worker(struct work_struct *); void rds_connect_complete(struct rds_connection *conn); diff --git a/net/rds/send.c b/net/rds/send.c index 65007113ffd95..399957e369f4d 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -1268,7 +1268,13 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) goto out; } else if (conn->c_base_conn->c_version == RDS_PROTOCOL_COMPAT_VERSION) { - conn = conn->c_base_conn; + if (!conn->c_reconnect || + conn->c_route_to_base) + conn = conn->c_base_conn; + else { + ret = -EAGAIN; + goto out; + } } } @@ -1597,3 +1603,25 @@ rds_send_hb(struct rds_connection *conn, int response) rds_message_put(rm); return 0; } + +void rds_route_to_base(struct rds_connection *conn) +{ + struct rds_message *rm, *tmp; + struct rds_connection *base_conn = conn->c_base_conn; + unsigned long flags; + + BUG_ON(!conn->c_tos || rds_conn_up(conn) || !base_conn || + !list_empty(&conn->c_retrans)); + + spin_lock_irqsave(&base_conn->c_lock, flags); + list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) { + list_del_init(&rm->m_conn_item); + rm->m_inc.i_conn = base_conn; + rm->m_inc.i_hdr.h_sequence = + cpu_to_be64(base_conn->c_next_tx_seq++); + list_add_tail(&rm->m_conn_item, &base_conn->c_send_queue); + } + spin_unlock_irqrestore(&base_conn->c_lock, flags); + conn->c_route_to_base = 1; + queue_delayed_work(rds_wq, &base_conn->c_send_w, 0); +} diff --git a/net/rds/threads.c b/net/rds/threads.c index 01a864ea3bb05..8935cc13c91e8 100644 --- a/net/rds/threads.c +++ b/net/rds/threads.c @@ -99,8 +99,8 @@ void rds_connect_complete(struct rds_connection *conn) conn->c_connection_start = get_seconds(); conn->c_reconnect = 1; - conn->c_committed_version = conn->c_version; conn->c_proposed_version = RDS_PROTOCOL_VERSION; + conn->c_route_to_base = 0; } EXPORT_SYMBOL_GPL(rds_connect_complete); @@ -224,6 +224,15 @@ void rds_recv_worker(struct work_struct *work) } } +void rds_reject_worker(struct work_struct *work) +{ + struct rds_connection *conn = container_of(work, struct rds_connection, c_reject_w.work); + + atomic_set(&conn->c_state, RDS_CONN_ERROR); + rds_conn_shutdown(conn, 0); + rds_route_to_base(conn); +} + void rds_hb_worker(struct work_struct *work) { struct rds_connection *conn = container_of(work, struct rds_connection, c_hb_w.work); @@ -259,7 +268,7 @@ void rds_shutdown_worker(struct work_struct *work) { struct rds_connection *conn = container_of(work, struct rds_connection, c_down_w); - rds_conn_shutdown(conn); + rds_conn_shutdown(conn, 1); } void rds_threads_exit(void) -- 2.50.1