RDS: IB: Drop unused and broken APM support

APM support in RDS has been broken and hence not being
used in production. We kept the code around but its time
to remove it and reduce the complexity in the RDS
failover code paths.

Orabug: 23027670

Tested-by: Michael Nowak <michael.nowak@oracle.com>
Tested-by: Rose Wang <rose.wang@oracle.com>
Tested-by: Rafael Alejandro Peralez <rafael.peralez@oracle.com>
Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>

diff --git a/net/rds/ib.c b/net/rds/ib.c
index 1e69e67885daaf4697e09086f03e16cd2359e2d2..3ade8031849e7353e888a8486a8922c15fa8bed6 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -53,18 +53,11 @@
 unsigned int rds_ib_fmr_1m_pool_size = RDS_FMR_1M_POOL_SIZE;
 unsigned int rds_ib_fmr_8k_pool_size = RDS_FMR_8K_POOL_SIZE;
 unsigned int rds_ib_retry_count = RDS_IB_DEFAULT_RETRY_COUNT;
-#if RDMA_RDS_APM_SUPPORTED
-unsigned int rds_ib_apm_enabled = 0;
-unsigned int rds_ib_apm_fallback = 1;
-#endif
 unsigned int rds_ib_active_bonding_enabled = 0;
 unsigned int rds_ib_active_bonding_fallback = 1;
 unsigned int rds_ib_active_bonding_reconnect_delay = 1;
 unsigned int rds_ib_active_bonding_trigger_delay_max_msecs; /* = 0; */
 unsigned int rds_ib_active_bonding_trigger_delay_min_msecs; /* = 0; */
-#if RDMA_RDS_APM_SUPPORTED
-unsigned int rds_ib_apm_timeout = RDS_IB_DEFAULT_TIMEOUT;
-#endif
 unsigned int rds_ib_rnr_retry_count = RDS_IB_DEFAULT_RNR_RETRY_COUNT;
 #if IB_RDS_CQ_VECTOR_SUPPORTED
 unsigned int rds_ib_cq_balance_enabled = 1;
@@ -79,22 +72,10 @@ module_param(rds_ib_fmr_8k_pool_size, int, 0444);
 MODULE_PARM_DESC(rds_ib_fmr_8k_pool_size, " Max number of 8k fmr per HCA");
 module_param(rds_ib_retry_count, int, 0444);
 MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
-#if RDMA_RDS_APM_SUPPORTED
-module_param(rds_ib_apm_enabled, int, 0444);
-MODULE_PARM_DESC(rds_ib_apm_enabled, " APM Enabled");
-#endif
 module_param(rds_ib_active_bonding_enabled, int, 0444);
 MODULE_PARM_DESC(rds_ib_active_bonding_enabled, " Active Bonding enabled");
-#if RDMA_RDS_APM_SUPPORTED
-module_param(rds_ib_apm_timeout, int, 0444);
-MODULE_PARM_DESC(rds_ib_apm_timeout, " APM timeout");
-#endif
 module_param(rds_ib_rnr_retry_count, int, 0444);
 MODULE_PARM_DESC(rds_ib_rnr_retry_count, " QP rnr retry count");
-#if RDMA_RDS_APM_SUPPORTED
-module_param(rds_ib_apm_fallback, int, 0444);
-MODULE_PARM_DESC(rds_ib_apm_fallback, " APM failback enabled");
-#endif
 module_param(rds_ib_active_bonding_fallback, int, 0444);
 MODULE_PARM_DESC(rds_ib_active_bonding_fallback, " Active Bonding failback Enabled");
 module_param(rds_ib_active_bonding_failover_groups, charp, 0444);
@@ -374,21 +355,11 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
                struct rdma_dev_addr *dev_addr;
 
                ic = conn->c_transport_data;
-#if RDMA_RDS_APM_SUPPORTED
-               if (rds_ib_apm_enabled) {
-                       memcpy((union ib_gid *) &iinfo->src_gid,
-                               &ic->i_cur_path.p_sgid, sizeof(union ib_gid));
-                       memcpy((union ib_gid *) &iinfo->dst_gid,
-                               &ic->i_cur_path.p_dgid, sizeof(union ib_gid));
-               } else
-#endif
-               {
-                       dev_addr = &ic->i_cm_id->route.addr.dev_addr;
-                       rdma_addr_get_sgid(dev_addr,
-                               (union ib_gid *) &iinfo->src_gid);
-                       rdma_addr_get_dgid(dev_addr,
-                               (union ib_gid *) &iinfo->dst_gid);
-               }
+               dev_addr = &ic->i_cm_id->route.addr.dev_addr;
+               rdma_addr_get_sgid(dev_addr,
+                       (union ib_gid *) &iinfo->src_gid);
+               rdma_addr_get_dgid(dev_addr,
+                       (union ib_gid *) &iinfo->dst_gid);
 
                rds_ibdev = ic->rds_ibdev;
                iinfo->max_send_wr = ic->i_send_ring.w_nr;
@@ -810,16 +781,6 @@ static int rds_ib_move_ip(char                     *from_dev,
                spin_lock_bh(&rds_ibdev->spinlock);
                list_for_each_entry(ic, &rds_ibdev->conn_list, ib_node) {
                        if (ic->conn->c_laddr == addr) {
-#if RDMA_RDS_APM_SUPPORTED
-                               if (rds_ib_apm_enabled) {
-                                       if (!memcmp(
-                                               &ic->i_cur_path.p_sgid,
-                                               &ip_config[to_port].gid,
-                                               sizeof(union ib_gid))) {
-                                               continue;
-                                       }
-                               }
-#endif
                                /* if local connection, update the ARP cache */
                                if (ic->conn->c_loopback) {
                                        for (i = 1; i <= ip_port_cnt; i++) {
@@ -2768,9 +2729,6 @@ struct rds_transport rds_ib_transport = {
        .sync_mr                = rds_ib_sync_mr,
        .free_mr                = rds_ib_free_mr,
        .flush_mrs              = rds_ib_flush_mrs,
-#if RDMA_RDS_APM_SUPPORTED
-       .check_migration        = rds_ib_check_migration,
-#endif
        .t_owner                = THIS_MODULE,
        .t_name                 = "infiniband",
        .t_type                 = RDS_TRANS_IB

diff --git a/net/rds/ib.h b/net/rds/ib.h
index 6351423c4827fe2a01977b6a79dd0657c9378d7e..f6b06641a88269f85066466949b3d4288bb5380e 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -227,13 +227,16 @@ struct rds_ib_connection {
 
        struct completion       i_last_wqe_complete;
 
+#if 0
        /* APM support */
-       struct rds_ib_migrate_work      i_migrate_w;
        struct rds_ib_path      i_pri_path;
        struct rds_ib_path      i_cur_path;
        unsigned int            i_alt_path_index;
-       unsigned int            i_active_side;
        unsigned long           i_last_migration;
+#endif
+       /* Active Bonding */
+       struct rds_ib_migrate_work      i_migrate_w;
+       unsigned int            i_active_side;
 
        int                     i_scq_vector;
        int                     i_rcq_vector;
@@ -498,7 +501,6 @@ struct rds_ib_statistics {
        uint64_t        s_ib_srq_lows;
        uint64_t        s_ib_srq_refills;
        uint64_t        s_ib_srq_empty_refills;
-       uint64_t        s_ib_failed_apm;
        uint64_t        s_ib_recv_added_to_cache;
        uint64_t        s_ib_recv_removed_from_cache;
 };
@@ -554,15 +556,8 @@ extern unsigned int rds_ib_fmr_1m_pool_size;
 extern unsigned int rds_ib_fmr_8k_pool_size;
 extern unsigned int rds_ib_retry_count;
 extern unsigned int rds_ib_rnr_retry_count;
-#if RDMA_RDS_APM_SUPPORTED
-extern unsigned int rds_ib_apm_enabled;
-extern unsigned int rds_ib_apm_fallback;
-#endif
 extern unsigned int rds_ib_active_bonding_enabled;
 extern unsigned int rds_ib_active_bonding_fallback;
-#if RDMA_RDS_APM_SUPPORTED
-extern unsigned int rds_ib_apm_timeout;
-#endif
 #if IB_RDS_CQ_VECTOR_SUPPORTED
 extern unsigned int rds_ib_cq_balance_enabled;
 #endif
@@ -588,10 +583,6 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
 int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id);
 void rds_ib_cm_connect_complete(struct rds_connection *conn,
                                struct rdma_cm_event *event);
-#if RDMA_RDS_APM_SUPPORTED
-void rds_ib_check_migration(struct rds_connection *conn,
-                               struct rdma_cm_event *event);
-#endif
 void rds_ib_init_frag(unsigned int version);
 
 #define rds_ib_conn_error(conn, fmt...) \

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 39c8b7e5f971e9c90cb23f57ba8e32c273881f86..4507e7ca8f18bb0e6a01cadb0ca51aaa6e75d5b0 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -301,41 +301,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even
        if (dp && dp->dp_ack_seq)
                rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL);
 
-#if RDMA_RDS_APM_SUPPORTED
-       if (rds_ib_apm_enabled) {
-               struct rdma_dev_addr *dev_addr;
-
-               dev_addr = &ic->i_cm_id->route.addr.dev_addr;
-
-               if (!ic->conn->c_reconnect) {
-                       rdma_addr_get_sgid(dev_addr,
-                               (union ib_gid *)&ic->i_pri_path.p_sgid);
-                       rdma_addr_get_dgid(dev_addr,
-                               (union ib_gid *)&ic->i_pri_path.p_dgid);
-                       printk(KERN_NOTICE "RDS/IB: connection "
-                               "<%u.%u.%u.%u,%u.%u.%u.%u,%d> primary path "
-                               "<"RDS_IB_GID_FMT","RDS_IB_GID_FMT">\n",
-                               NIPQUAD(conn->c_laddr),
-                               NIPQUAD(conn->c_faddr),
-                               conn->c_tos,
-                               RDS_IB_GID_ARG(ic->i_pri_path.p_sgid),
-                               RDS_IB_GID_ARG(ic->i_pri_path.p_dgid));
-               }
-               rdma_addr_get_sgid(dev_addr,
-                       (union ib_gid *)&ic->i_cur_path.p_sgid);
-               rdma_addr_get_dgid(dev_addr,
-                       (union ib_gid *)&ic->i_cur_path.p_dgid);
-       }
-#endif
-
        rds_connect_complete(conn);
-
-#if RDMA_RDS_APM_SUPPORTED
-        if (ic->i_last_migration) {
-                rds_ib_stats_inc(s_ib_failed_apm);
-                ic->i_last_migration = 0;
-        }
-#endif
 }
 
 static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
@@ -563,6 +529,7 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
                complete(&ic->i_last_wqe_complete);
                break;
        case IB_EVENT_PATH_MIG:
+#if 0
                memcpy(&ic->i_cur_path.p_sgid,
                        &ic->i_cm_id->route.path_rec[ic->i_alt_path_index].sgid,
                        sizeof(union ib_gid));
@@ -596,6 +563,7 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
                                RDS_IB_GID_ARG(ic->i_cur_path.p_dgid));
                }
                ic->i_last_migration = get_seconds();
+#endif
 
                break;
        case IB_EVENT_PATH_MIG_ERR:
@@ -1012,23 +980,12 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
                event->param.conn.initiator_depth,
                ib_init_frag_size);
 
-#if RDMA_RDS_APM_SUPPORTED
-       if (rds_ib_apm_enabled)
-               rdma_set_timeout(cm_id, rds_ib_apm_timeout);
-#endif
        /* rdma_accept() calls rdma_reject() internally if it fails */
        err = rdma_accept(cm_id, &conn_param);
        if (err) {
                conn->c_drop_source = DR_IB_RDMA_ACCEPT_FAIL;
                rds_ib_conn_error(conn, "rdma_accept failed (%d)\n", err);
        }
-#if RDMA_RDS_APM_SUPPORTED
-       else if (rds_ib_apm_enabled && !conn->c_loopback) {
-               err = rdma_enable_apm(cm_id, RDMA_ALT_PATH_BEST);
-               if (err)
-                       printk(KERN_WARNING "RDS/IB: APM couldn't be enabled for passive side: %d\n", err);
-       }
-#endif
 
 out:
        if (conn)
@@ -1047,14 +1004,6 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
        struct rds_ib_connect_private dp;
        int ret;
 
-#if RDMA_RDS_APM_SUPPORTED
-       if (rds_ib_apm_enabled && !conn->c_loopback) {
-               ret = rdma_enable_apm(cm_id, RDMA_ALT_PATH_BEST);
-               if (ret)
-                       printk(KERN_WARNING "RDS/IB: APM couldn't be enabled for active side: %d\n", ret);
-       }
-#endif
-
        rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
        ic->i_flowctl = rds_ib_sysctl_flow_control;     /* advertise flow control */
        /* Use ic->i_flowctl as the first post credit to enable
@@ -1108,11 +1057,6 @@ static void rds_ib_migrate(struct work_struct *_work)
        struct rdma_cm_id *cm_id = ic->i_cm_id;
        int ret = 0;
 
-#if RDMA_RDS_APM_SUPPORTED
-       if (!rds_ib_apm_fallback)
-               return;
-#endif
-
        if (!ic->i_active_side) {
                ret = ib_query_qp(cm_id->qp, &qp_attr, IB_QP_PATH_MIG_STATE,
                                &qp_init_attr);
@@ -1140,63 +1084,6 @@ static void rds_ib_migrate(struct work_struct *_work)
        }
 }
 
-#if RDMA_RDS_APM_SUPPORTED
-void rds_ib_check_migration(struct rds_connection *conn,
-                       struct rdma_cm_event *event)
-{
-       struct rds_ib_connection *ic = conn->c_transport_data;
-       union ib_gid sgid;
-       union ib_gid dgid;
-       struct ib_qp_init_attr qp_init_attr;
-       struct ib_qp_attr qp_attr;
-       struct rdma_cm_id *cm_id = ic->i_cm_id;
-       int err;
-
-       if (!rds_ib_apm_enabled || !rds_conn_up(ic->conn))
-               return ;
-
-       ic->i_alt_path_index = event->param.ud.alt_path_index;
-
-       memcpy(&sgid, &cm_id->route.path_rec[event->param.ud.alt_path_index].
-               sgid, sizeof(union ib_gid));
-       memcpy(&dgid, &cm_id->route.path_rec[event->param.ud.alt_path_index].
-               dgid, sizeof(union ib_gid));
-
-       printk(KERN_NOTICE
-               "RDS/IB: connection "
-               "<%u.%u.%u.%u,%u.%u.%u.%u,%d> loaded alternate path "
-               "<"RDS_IB_GID_FMT","RDS_IB_GID_FMT">\n",
-               NIPQUAD(conn->c_laddr),
-               NIPQUAD(conn->c_faddr),
-               conn->c_tos,
-               RDS_IB_GID_ARG(sgid), RDS_IB_GID_ARG(dgid));
-
-       err = ib_query_qp(cm_id->qp, &qp_attr, IB_QP_ALT_PATH, &qp_init_attr);
-       if (err) {
-               printk(KERN_ERR "RDS/IB: ib_query_qp failed (%d)\n", err);
-               return;
-       }
-       qp_attr.alt_timeout = rds_ib_apm_timeout;
-       err = ib_modify_qp(cm_id->qp, &qp_attr, IB_QP_ALT_PATH);
-       if (err) {
-               printk(KERN_ERR "RDS/IB: ib_modify_qp failed (%d)\n", err);
-               return;
-       }
-
-       if (!memcmp(&ic->i_pri_path.p_sgid, &sgid, sizeof(union ib_gid)) &&
-               !memcmp(&ic->i_pri_path.p_dgid, &dgid, sizeof(union ib_gid))) {
-               if (memcmp(&ic->i_cur_path.p_sgid, &ic->i_pri_path.p_sgid,
-                               sizeof(union ib_gid)) ||
-                       memcmp(&ic->i_cur_path.p_dgid, &ic->i_pri_path.p_dgid,
-                               sizeof(union ib_gid))) {
-
-                       ic->i_migrate_w.ic = ic;
-                       queue_delayed_work(rds_wq, &ic->i_migrate_w.work, 0);
-               }
-       }
-}
-#endif
-
 int rds_ib_conn_connect(struct rds_connection *conn)
 {
        struct rds_ib_connection *ic = conn->c_transport_data;

diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index 38466937363512fb574241dde230001bc305ae45..c4298c50323781060ff238a92ea095e5ec02503f 100644 (file)
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -350,8 +350,7 @@ void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)
                        conn->c_tos, wc->status, wc->vendor_err);
                rds_rtd(RDS_RTD_ERR, "status %u => %s\n", wc->status,
                        rds_ib_wc_status_str(wc->status));
-       } else
-               ic->i_last_migration = 0;
+       }
 }
 
 /*

diff --git a/net/rds/ib_stats.c b/net/rds/ib_stats.c
index 2be25a21cdc2317954eab026439794706529c9ec..96c78598cccd135da867687071dd3a301fc4521c 100644 (file)
--- a/net/rds/ib_stats.c
+++ b/net/rds/ib_stats.c
@@ -80,7 +80,6 @@ static char *rds_ib_stat_names[] = {
        "ib_srq_lows",
        "ib_srq_refills",
        "ib_srq_empty_refills",
-       "ib_apm_reconnect",
        "ib_recv_cache_added",
        "ib_recv_cache_removed",
 };

diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index 483905d58eca911f4be0e9ff0e22740c99934671..0c8e34566dc2bf8247fc51efb3df07968d488992 100644 (file)
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -71,12 +71,6 @@ static char *rds_cm_event_strings[] = {
        RDS_CM_EVENT_STRING(MULTICAST_ERROR),
        RDS_CM_EVENT_STRING(ADDR_CHANGE),
        RDS_CM_EVENT_STRING(TIMEWAIT_EXIT),
-#if RDMA_RDS_APM_SUPPORTED
-       RDS_CM_EVENT_STRING(ALT_ROUTE_RESOLVED),
-       RDS_CM_EVENT_STRING(ALT_ROUTE_ERROR),
-       RDS_CM_EVENT_STRING(LOAD_ALT_PATH),
-       RDS_CM_EVENT_STRING(ALT_PATH_LOADED),
-#endif
 #undef RDS_CM_EVENT_STRING
 };
 
@@ -131,11 +125,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
        case RDMA_CM_EVENT_ADDR_RESOLVED:
                rdma_set_service_type(cm_id, conn->c_tos);
 
-#if RDMA_RDS_APM_SUPPORTED
-               if (rds_ib_apm_enabled)
-                       rdma_set_timeout(cm_id, rds_ib_apm_timeout);
-#endif
-
                if (conn->c_tos && conn->c_reconnect) {
                        struct rds_ib_connection *base_ic =
                                conn->c_base_conn->c_transport_data;
@@ -221,22 +210,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
                }
                break;
 
-#if RDMA_RDS_APM_SUPPORTED
-       case RDMA_CM_EVENT_ALT_PATH_LOADED:
-               rdsdebug("RDS: alt path loaded\n");
-               if (conn)
-                       trans->check_migration(conn, event);
-               break;
-
-       case RDMA_CM_EVENT_ALT_ROUTE_RESOLVED:
-               rdsdebug("RDS: alt route resolved\n");
-               break;
-
-       case RDMA_CM_EVENT_ALT_ROUTE_ERROR:
-               rdsdebug("RDS: alt route resolve error\n");
-               break;
-#endif
-
        case RDMA_CM_EVENT_ROUTE_ERROR:
                /* IP might have been moved so flush the ARP entry and retry */
                page = alloc_page(GFP_HIGHUSER);
@@ -339,16 +312,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
                        "ADDR_CHANGE event <%u.%u.%u.%u,%u.%u.%u.%u>\n",
                        NIPQUAD(conn->c_laddr),
                        NIPQUAD(conn->c_faddr));
-#if RDMA_RDS_APM_SUPPORTED
-               if (conn && !rds_ib_apm_enabled) {
-                       rds_rtd(RDS_RTD_CM,
-                               "ADDR_CHANGE: calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
-                               NIPQUAD(conn->c_laddr), NIPQUAD(conn->c_faddr),
-                               conn->c_tos);
-                       conn->c_drop_source = DR_IB_ADDR_CHANGE;
-                       rds_conn_drop(conn);
-               }
-#else
                if (conn) {
                        rds_rtd(RDS_RTD_CM,
                                "ADDR_CHANGE: calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
@@ -357,7 +320,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
                        conn->c_drop_source = DR_IB_ADDR_CHANGE;
                        rds_conn_drop(conn);
                }
-#endif
                break;
 
        case RDMA_CM_EVENT_DISCONNECTED:

diff --git a/net/rds/rds.h b/net/rds/rds.h
index f856c55b8acab731989d50859f97bdbb756266dd..6978f0db842436c7d43b5c5fb7da2006d3f5f0b1 100644 (file)
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -26,7 +26,6 @@
 #define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
 
 #define IB_RDS_CQ_VECTOR_SUPPORTED 0
-#define RDMA_RDS_APM_SUPPORTED 0
 
 /*
  * XXX randomly chosen, but at least seems to be unused: