RDMA CM: Add reason code for IB_CM_REJ_CONSUMER_DEFINED

RDS: Support rolling downgrade from version 4.1 to 3.1

Orabug: 17484682

Signed-off-by: Giri Adari <giri.adari@oracle.com>
Signed-off-by: Richard Frank <richard.frank@oracle.com>
Signed-off-by: Chien-Hua Yen <chien.yen@oracle.com>
(cherry picked from commit 7b66ddd7f6a5b023191d74949fab41af245775a3)
Signed-off-by: Jerry Snitselaar <jerry.snitselaar@oracle.com>
Conflicts:
	net/rds/rds.h
(cherry picked from commit 0373566ba0d74f655ae83e09748f7cc8d553f351)

diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 0271608a51c40ff2ade721f94b710997ffbe5c2d..2831387a4eb72156c02fe8c758f7131a3e91685c 100644 (file)
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -876,8 +876,10 @@ retest:
                        spin_unlock_irq(&cm_id_priv->lock);
                } else {
                        spin_unlock_irq(&cm_id_priv->lock);
+                       if (!err)
+                               err = -EINVAL;
                        ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
-                                      NULL, 0, NULL, 0);
+                                       NULL, 0, &err, sizeof(err));
                }
                break;
        case IB_CM_REP_SENT:
@@ -888,8 +890,10 @@ retest:
        case IB_CM_REP_RCVD:
        case IB_CM_MRA_REP_SENT:
                spin_unlock_irq(&cm_id_priv->lock);
+               if (!err)
+                       err = -EINVAL;
                ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
-                              NULL, 0, NULL, 0);
+                               NULL, 0, &err, sizeof(err));
                break;
        case IB_CM_ESTABLISHED:
                spin_unlock_irq(&cm_id_priv->lock);

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 38ffe098150351aef9ff2ac650726ae7926f6856..3eef7e352a7280eb330566ed07950781260a82c5 100644 (file)
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -1102,7 +1102,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv)
 reject:
        cma_modify_qp_err(id_priv);
        ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
-                      NULL, 0, NULL, 0);
+                       NULL, 0, &ret, sizeof(int));
        return ret;
 }
 
@@ -3049,7 +3049,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
        return 0;
 reject:
        cma_modify_qp_err(id_priv);
-       rdma_reject(id, NULL, 0);
+       rdma_reject(id, &ret, sizeof(int));
        return ret;
 }
 EXPORT_SYMBOL(rdma_accept);

diff --git a/net/rds/connection.c b/net/rds/connection.c
index 069ffcf1002b9e48155686b74e0f204fb81b0420..3c3283c3dd90335828fcc5a4c1a18b88aa2992e5 100644 (file)
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -214,6 +214,7 @@ static struct rds_connection *__rds_conn_create(__be32 laddr, __be32 faddr,
        INIT_DELAYED_WORK(&conn->c_recv_w, rds_recv_worker);
        INIT_DELAYED_WORK(&conn->c_conn_w, rds_connect_worker);
        INIT_DELAYED_WORK(&conn->c_hb_w, rds_hb_worker);
+       INIT_DELAYED_WORK(&conn->c_reject_w, rds_reject_worker);
        INIT_WORK(&conn->c_down_w, rds_shutdown_worker);
        mutex_init(&conn->c_cm_lock);
        conn->c_flags = 0;
@@ -293,7 +294,7 @@ struct rds_connection *rds_conn_find(__be32 laddr, __be32 faddr,
 }
 EXPORT_SYMBOL_GPL(rds_conn_find);
 
-void rds_conn_shutdown(struct rds_connection *conn)
+void rds_conn_shutdown(struct rds_connection *conn, int restart)
 {
        /* shut it down unless it's down already */
        if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_DOWN)) {
@@ -343,7 +344,7 @@ void rds_conn_shutdown(struct rds_connection *conn)
         * conn - the reconnect is always triggered by the active peer. */
        cancel_delayed_work_sync(&conn->c_conn_w);
        rcu_read_lock();
-       if (!hlist_unhashed(&conn->c_hash_node)) {
+       if (!hlist_unhashed(&conn->c_hash_node) && restart) {
                rcu_read_unlock();
                rds_queue_reconnect(conn);
        } else {

diff --git a/net/rds/ib.c b/net/rds/ib.c
index 0acf30cc636f31c9c573daca0a12808d4959c1c5..628430bdd83a191e0e7a54a36f2ee4bf917bfcb4 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -295,7 +295,7 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
                                    void *buffer)
 {
        struct rds_info_rdma_connection *iinfo = buffer;
-       struct rds_ib_connection *ic;
+       struct rds_ib_connection *ic = conn->c_transport_data;
 
        /* We will only ever look at IB transports */
        if (conn->c_trans != &rds_ib_transport)
@@ -306,6 +306,12 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
 
        memset(&iinfo->src_gid, 0, sizeof(iinfo->src_gid));
        memset(&iinfo->dst_gid, 0, sizeof(iinfo->dst_gid));
+
+       if (ic) {
+               iinfo->tos = conn->c_tos;
+               iinfo->sl = ic->i_sl;
+       }
+
        if (rds_conn_state(conn) == RDS_CONN_UP) {
                struct rds_ib_device *rds_ibdev;
                struct rdma_dev_addr *dev_addr;
@@ -332,8 +338,6 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
                iinfo->max_recv_wr = ic->i_recv_ring.w_nr;
                iinfo->max_send_sge = rds_ibdev->max_sge;
                rds_ib_get_mr_info(rds_ibdev, iinfo);
-               iinfo->tos = ic->conn->c_tos;
-               iinfo->sl = ic->i_sl;
                iinfo->cache_allocs = atomic_read(&ic->i_cache_allocs);
        }
        return 1;

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 083425602e0d4881bb9d97566c93905e5988646a..2c97d639acb0ead95d6e24bf515c9bff324d1ae9 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -732,20 +732,13 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
                goto out;
        }
 
-       if (conn->c_reconnect && (conn->c_version > version)) {
-               printk(KERN_WARNING "RDS/IB: connection "
-                       "<%u.%u.%u.%u,%u.%u.%u.%u,%d,%u.%u> rejecting version "
-                       "(%u/%u)\n",
-                       NIPQUAD(conn->c_laddr),
-                       NIPQUAD(conn->c_faddr),
-                       conn->c_tos,
-                       RDS_PROTOCOL_MAJOR(conn->c_version),
-                       RDS_PROTOCOL_MINOR(conn->c_version),
-                       RDS_PROTOCOL_MAJOR(version),
-                       RDS_PROTOCOL_MINOR(version));
-
-               conn = NULL;
-               goto out;
+       if (dp->dp_tos && !conn->c_base_conn) {
+               conn->c_base_conn = rds_conn_create(dp->dp_daddr, dp->dp_saddr,
+                                       &rds_ib_transport, 0, GFP_KERNEL);
+               if (IS_ERR(conn->c_base_conn)) {
+                       conn = NULL;
+                       goto out;
+               }
        }
 
        /*
@@ -872,8 +865,6 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
        }
 #endif
 
-       /* If the peer doesn't do protocol negotiation, we must
-        * default to RDSv3.0 */
        rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
        ic->i_flowctl = rds_ib_sysctl_flow_control;     /* advertise flow control */
 

diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index a97c862aa54bdd9e954f83895a446b9e5ba54fac..9e7c4bcf89c000ee1e3790ddee1d97a257d79502 100644 (file)
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -186,23 +186,19 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
        case RDMA_CM_EVENT_REJECTED:
                err = (int *)event->param.conn.private_data;
                if (conn) {
-                       if (!conn->c_reconnect || conn->c_committed_version ==
-                               RDS_PROTOCOL_COMPAT_VERSION) {
-                               if ((*err) == 0 &&
-                                       event->status == RDS_REJ_CONSUMER_DEFINED) {
-                                       /* rejection from 3.x protocol */
-                                       if (!conn->c_tos) {
-                                               /* retry the connect with a
-                                                  lower compatible protocol */
-                                               conn->c_proposed_version =
-                                                       RDS_PROTOCOL_COMPAT_VERSION;
-                                       }
-                               } else {
+                       if (event->status == RDS_REJ_CONSUMER_DEFINED && (*err) == 0) {
+                               /* Rejection from RDSV3.1 */
+                               if (!conn->c_tos) {
                                        conn->c_proposed_version =
-                                               RDS_PROTOCOL_VERSION;
+                                               RDS_PROTOCOL_COMPAT_VERSION;
+                                       rds_conn_drop(conn);
+                               } else {
+                                       queue_delayed_work(rds_wq,
+                                               &conn->c_reject_w,
+                                               msecs_to_jiffies(10));
                                }
-                       }
-                       rds_conn_drop(conn);
+                       } else
+                               rds_conn_drop(conn);
                }
                break;
 

diff --git a/net/rds/rds.h b/net/rds/rds.h
index 7c1a06ea2b45a9bce4d8b505c42fdf6d7a3f5141..4e747773f449538b6e1f1635858f07d5cefbc099 100644 (file)
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -128,6 +128,7 @@ struct rds_connection {
        struct delayed_work     c_send_w;
        struct delayed_work     c_recv_w;
        struct delayed_work     c_conn_w;
+       struct delayed_work     c_reject_w;
        struct delayed_work     c_hb_w;
        struct work_struct      c_down_w;
        struct mutex            c_cm_lock;      /* protect conn state & cm */
@@ -142,7 +143,6 @@ struct rds_connection {
 
        /* Protocol version */
        unsigned int            c_proposed_version;
-       unsigned int            c_committed_version;
        unsigned int            c_version;
 
        /* Re-connect stall diagnostics */
@@ -162,6 +162,8 @@ struct rds_connection {
        unsigned long           c_hb_start;
 
        struct rds_connection   *c_base_conn;
+
+       unsigned int            c_route_to_base;
 };
 
 #define RDS_FLAG_CONG_BITMAP   0x01
@@ -670,7 +672,7 @@ struct rds_connection *rds_conn_create_outgoing(__be32 laddr, __be32 faddr,
                                u8 tos, gfp_t gfp);
 struct rds_connection *rds_conn_find(__be32 laddr, __be32 faddr,
                                        struct rds_transport *trans, u8 tos);
-void rds_conn_shutdown(struct rds_connection *conn);
+void rds_conn_shutdown(struct rds_connection *conn, int restart);
 void rds_conn_destroy(struct rds_connection *conn);
 void rds_conn_reset(struct rds_connection *conn);
 void rds_conn_drop(struct rds_connection *conn);
@@ -788,6 +790,7 @@ struct rds_message *rds_send_get_message(struct rds_connection *,
                                         struct rm_rdma_op *);
 int rds_send_internal(struct rds_connection *conn, struct rds_sock *rs,
                      struct sk_buff *skb, gfp_t gfp);
+void rds_route_to_base(struct rds_connection *conn);
 
 extern unsigned int rds_async_send_enabled;
 
@@ -860,6 +863,7 @@ void rds_queue_reconnect(struct rds_connection *conn);
 void rds_connect_worker(struct work_struct *);
 void rds_shutdown_worker(struct work_struct *);
 void rds_send_worker(struct work_struct *);
+void rds_reject_worker(struct work_struct *);
 void rds_recv_worker(struct work_struct *);
 void rds_hb_worker(struct work_struct *);
 void rds_connect_complete(struct rds_connection *conn);

diff --git a/net/rds/send.c b/net/rds/send.c
index 65007113ffd959e9918ad1e4abb40affe6e275d9..399957e369f4dce99cdca1835a6ba3e007a998cc 100644 (file)
--- a/net/rds/send.c
+++ b/net/rds/send.c
@@ -1268,7 +1268,13 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
                        goto out;
                } else if (conn->c_base_conn->c_version ==
                                RDS_PROTOCOL_COMPAT_VERSION) {
-                       conn = conn->c_base_conn;
+                       if (!conn->c_reconnect ||
+                               conn->c_route_to_base)
+                               conn = conn->c_base_conn;
+                       else {
+                               ret = -EAGAIN;
+                               goto out;
+                       }
                }
        }
 
@@ -1597,3 +1603,25 @@ rds_send_hb(struct rds_connection *conn, int response)
        rds_message_put(rm);
        return 0;
 }
+
+void rds_route_to_base(struct rds_connection *conn)
+{
+       struct rds_message *rm, *tmp;
+       struct rds_connection *base_conn = conn->c_base_conn;
+       unsigned long flags;
+
+       BUG_ON(!conn->c_tos || rds_conn_up(conn) || !base_conn ||
+               !list_empty(&conn->c_retrans));
+
+       spin_lock_irqsave(&base_conn->c_lock, flags);
+       list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) {
+               list_del_init(&rm->m_conn_item);
+               rm->m_inc.i_conn = base_conn;
+               rm->m_inc.i_hdr.h_sequence =
+                       cpu_to_be64(base_conn->c_next_tx_seq++);
+               list_add_tail(&rm->m_conn_item, &base_conn->c_send_queue);
+       }
+       spin_unlock_irqrestore(&base_conn->c_lock, flags);
+       conn->c_route_to_base = 1;
+       queue_delayed_work(rds_wq, &base_conn->c_send_w, 0);
+}

diff --git a/net/rds/threads.c b/net/rds/threads.c
index 01a864ea3bb0580f6efcfd3b085681fc447cf272..8935cc13c91e83b33cce12a6a7718df7750db75e 100644 (file)
--- a/net/rds/threads.c
+++ b/net/rds/threads.c
@@ -99,8 +99,8 @@ void rds_connect_complete(struct rds_connection *conn)
 
        conn->c_connection_start = get_seconds();
        conn->c_reconnect = 1;
-       conn->c_committed_version = conn->c_version;
        conn->c_proposed_version = RDS_PROTOCOL_VERSION;
+       conn->c_route_to_base = 0;
 }
 EXPORT_SYMBOL_GPL(rds_connect_complete);
 
@@ -224,6 +224,15 @@ void rds_recv_worker(struct work_struct *work)
        }
 }
 
+void rds_reject_worker(struct work_struct *work)
+{
+       struct rds_connection *conn = container_of(work, struct rds_connection, c_reject_w.work);
+
+       atomic_set(&conn->c_state, RDS_CONN_ERROR);
+       rds_conn_shutdown(conn, 0);
+       rds_route_to_base(conn);
+}
+
 void rds_hb_worker(struct work_struct *work)
 {
        struct rds_connection *conn = container_of(work, struct rds_connection, c_hb_w.work);
@@ -259,7 +268,7 @@ void rds_shutdown_worker(struct work_struct *work)
 {
        struct rds_connection *conn = container_of(work, struct rds_connection, c_down_w);
 
-       rds_conn_shutdown(conn);
+       rds_conn_shutdown(conn, 1);
 }
 
 void rds_threads_exit(void)