From: Santosh Shilimkar Date: Wed, 10 Feb 2016 17:08:43 +0000 (-0800) Subject: RDS: IB: Drop unused and broken APM support X-Git-Tag: v4.1.12-92~153^2~2 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=321fc0bc396b89251aa8b77c991e88a8fd827151;p=users%2Fjedix%2Flinux-maple.git RDS: IB: Drop unused and broken APM support APM support in RDS has been broken and hence not being used in production. We kept the code around but its time to remove it and reduce the complexity in the RDS failover code paths. Orabug: 23027670 Tested-by: Michael Nowak Tested-by: Rose Wang Tested-by: Rafael Alejandro Peralez Signed-off-by: Santosh Shilimkar --- diff --git a/net/rds/ib.c b/net/rds/ib.c index 1e69e67885da..3ade8031849e 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -53,18 +53,11 @@ unsigned int rds_ib_fmr_1m_pool_size = RDS_FMR_1M_POOL_SIZE; unsigned int rds_ib_fmr_8k_pool_size = RDS_FMR_8K_POOL_SIZE; unsigned int rds_ib_retry_count = RDS_IB_DEFAULT_RETRY_COUNT; -#if RDMA_RDS_APM_SUPPORTED -unsigned int rds_ib_apm_enabled = 0; -unsigned int rds_ib_apm_fallback = 1; -#endif unsigned int rds_ib_active_bonding_enabled = 0; unsigned int rds_ib_active_bonding_fallback = 1; unsigned int rds_ib_active_bonding_reconnect_delay = 1; unsigned int rds_ib_active_bonding_trigger_delay_max_msecs; /* = 0; */ unsigned int rds_ib_active_bonding_trigger_delay_min_msecs; /* = 0; */ -#if RDMA_RDS_APM_SUPPORTED -unsigned int rds_ib_apm_timeout = RDS_IB_DEFAULT_TIMEOUT; -#endif unsigned int rds_ib_rnr_retry_count = RDS_IB_DEFAULT_RNR_RETRY_COUNT; #if IB_RDS_CQ_VECTOR_SUPPORTED unsigned int rds_ib_cq_balance_enabled = 1; @@ -79,22 +72,10 @@ module_param(rds_ib_fmr_8k_pool_size, int, 0444); MODULE_PARM_DESC(rds_ib_fmr_8k_pool_size, " Max number of 8k fmr per HCA"); module_param(rds_ib_retry_count, int, 0444); MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error"); -#if RDMA_RDS_APM_SUPPORTED -module_param(rds_ib_apm_enabled, int, 0444); -MODULE_PARM_DESC(rds_ib_apm_enabled, " APM Enabled"); -#endif module_param(rds_ib_active_bonding_enabled, int, 0444); MODULE_PARM_DESC(rds_ib_active_bonding_enabled, " Active Bonding enabled"); -#if RDMA_RDS_APM_SUPPORTED -module_param(rds_ib_apm_timeout, int, 0444); -MODULE_PARM_DESC(rds_ib_apm_timeout, " APM timeout"); -#endif module_param(rds_ib_rnr_retry_count, int, 0444); MODULE_PARM_DESC(rds_ib_rnr_retry_count, " QP rnr retry count"); -#if RDMA_RDS_APM_SUPPORTED -module_param(rds_ib_apm_fallback, int, 0444); -MODULE_PARM_DESC(rds_ib_apm_fallback, " APM failback enabled"); -#endif module_param(rds_ib_active_bonding_fallback, int, 0444); MODULE_PARM_DESC(rds_ib_active_bonding_fallback, " Active Bonding failback Enabled"); module_param(rds_ib_active_bonding_failover_groups, charp, 0444); @@ -374,21 +355,11 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn, struct rdma_dev_addr *dev_addr; ic = conn->c_transport_data; -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled) { - memcpy((union ib_gid *) &iinfo->src_gid, - &ic->i_cur_path.p_sgid, sizeof(union ib_gid)); - memcpy((union ib_gid *) &iinfo->dst_gid, - &ic->i_cur_path.p_dgid, sizeof(union ib_gid)); - } else -#endif - { - dev_addr = &ic->i_cm_id->route.addr.dev_addr; - rdma_addr_get_sgid(dev_addr, - (union ib_gid *) &iinfo->src_gid); - rdma_addr_get_dgid(dev_addr, - (union ib_gid *) &iinfo->dst_gid); - } + dev_addr = &ic->i_cm_id->route.addr.dev_addr; + rdma_addr_get_sgid(dev_addr, + (union ib_gid *) &iinfo->src_gid); + rdma_addr_get_dgid(dev_addr, + (union ib_gid *) &iinfo->dst_gid); rds_ibdev = ic->rds_ibdev; iinfo->max_send_wr = ic->i_send_ring.w_nr; @@ -810,16 +781,6 @@ static int rds_ib_move_ip(char *from_dev, spin_lock_bh(&rds_ibdev->spinlock); list_for_each_entry(ic, &rds_ibdev->conn_list, ib_node) { if (ic->conn->c_laddr == addr) { -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled) { - if (!memcmp( - &ic->i_cur_path.p_sgid, - &ip_config[to_port].gid, - sizeof(union ib_gid))) { - continue; - } - } -#endif /* if local connection, update the ARP cache */ if (ic->conn->c_loopback) { for (i = 1; i <= ip_port_cnt; i++) { @@ -2768,9 +2729,6 @@ struct rds_transport rds_ib_transport = { .sync_mr = rds_ib_sync_mr, .free_mr = rds_ib_free_mr, .flush_mrs = rds_ib_flush_mrs, -#if RDMA_RDS_APM_SUPPORTED - .check_migration = rds_ib_check_migration, -#endif .t_owner = THIS_MODULE, .t_name = "infiniband", .t_type = RDS_TRANS_IB diff --git a/net/rds/ib.h b/net/rds/ib.h index 6351423c4827..f6b06641a882 100644 --- a/net/rds/ib.h +++ b/net/rds/ib.h @@ -227,13 +227,16 @@ struct rds_ib_connection { struct completion i_last_wqe_complete; +#if 0 /* APM support */ - struct rds_ib_migrate_work i_migrate_w; struct rds_ib_path i_pri_path; struct rds_ib_path i_cur_path; unsigned int i_alt_path_index; - unsigned int i_active_side; unsigned long i_last_migration; +#endif + /* Active Bonding */ + struct rds_ib_migrate_work i_migrate_w; + unsigned int i_active_side; int i_scq_vector; int i_rcq_vector; @@ -498,7 +501,6 @@ struct rds_ib_statistics { uint64_t s_ib_srq_lows; uint64_t s_ib_srq_refills; uint64_t s_ib_srq_empty_refills; - uint64_t s_ib_failed_apm; uint64_t s_ib_recv_added_to_cache; uint64_t s_ib_recv_removed_from_cache; }; @@ -554,15 +556,8 @@ extern unsigned int rds_ib_fmr_1m_pool_size; extern unsigned int rds_ib_fmr_8k_pool_size; extern unsigned int rds_ib_retry_count; extern unsigned int rds_ib_rnr_retry_count; -#if RDMA_RDS_APM_SUPPORTED -extern unsigned int rds_ib_apm_enabled; -extern unsigned int rds_ib_apm_fallback; -#endif extern unsigned int rds_ib_active_bonding_enabled; extern unsigned int rds_ib_active_bonding_fallback; -#if RDMA_RDS_APM_SUPPORTED -extern unsigned int rds_ib_apm_timeout; -#endif #if IB_RDS_CQ_VECTOR_SUPPORTED extern unsigned int rds_ib_cq_balance_enabled; #endif @@ -588,10 +583,6 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id); void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event); -#if RDMA_RDS_APM_SUPPORTED -void rds_ib_check_migration(struct rds_connection *conn, - struct rdma_cm_event *event); -#endif void rds_ib_init_frag(unsigned int version); #define rds_ib_conn_error(conn, fmt...) \ diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 39c8b7e5f971..4507e7ca8f18 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -301,41 +301,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even if (dp && dp->dp_ack_seq) rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL); -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled) { - struct rdma_dev_addr *dev_addr; - - dev_addr = &ic->i_cm_id->route.addr.dev_addr; - - if (!ic->conn->c_reconnect) { - rdma_addr_get_sgid(dev_addr, - (union ib_gid *)&ic->i_pri_path.p_sgid); - rdma_addr_get_dgid(dev_addr, - (union ib_gid *)&ic->i_pri_path.p_dgid); - printk(KERN_NOTICE "RDS/IB: connection " - "<%u.%u.%u.%u,%u.%u.%u.%u,%d> primary path " - "<"RDS_IB_GID_FMT","RDS_IB_GID_FMT">\n", - NIPQUAD(conn->c_laddr), - NIPQUAD(conn->c_faddr), - conn->c_tos, - RDS_IB_GID_ARG(ic->i_pri_path.p_sgid), - RDS_IB_GID_ARG(ic->i_pri_path.p_dgid)); - } - rdma_addr_get_sgid(dev_addr, - (union ib_gid *)&ic->i_cur_path.p_sgid); - rdma_addr_get_dgid(dev_addr, - (union ib_gid *)&ic->i_cur_path.p_dgid); - } -#endif - rds_connect_complete(conn); - -#if RDMA_RDS_APM_SUPPORTED - if (ic->i_last_migration) { - rds_ib_stats_inc(s_ib_failed_apm); - ic->i_last_migration = 0; - } -#endif } static void rds_ib_cm_fill_conn_param(struct rds_connection *conn, @@ -563,6 +529,7 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data) complete(&ic->i_last_wqe_complete); break; case IB_EVENT_PATH_MIG: +#if 0 memcpy(&ic->i_cur_path.p_sgid, &ic->i_cm_id->route.path_rec[ic->i_alt_path_index].sgid, sizeof(union ib_gid)); @@ -596,6 +563,7 @@ static void rds_ib_qp_event_handler(struct ib_event *event, void *data) RDS_IB_GID_ARG(ic->i_cur_path.p_dgid)); } ic->i_last_migration = get_seconds(); +#endif break; case IB_EVENT_PATH_MIG_ERR: @@ -1012,23 +980,12 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, event->param.conn.initiator_depth, ib_init_frag_size); -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled) - rdma_set_timeout(cm_id, rds_ib_apm_timeout); -#endif /* rdma_accept() calls rdma_reject() internally if it fails */ err = rdma_accept(cm_id, &conn_param); if (err) { conn->c_drop_source = DR_IB_RDMA_ACCEPT_FAIL; rds_ib_conn_error(conn, "rdma_accept failed (%d)\n", err); } -#if RDMA_RDS_APM_SUPPORTED - else if (rds_ib_apm_enabled && !conn->c_loopback) { - err = rdma_enable_apm(cm_id, RDMA_ALT_PATH_BEST); - if (err) - printk(KERN_WARNING "RDS/IB: APM couldn't be enabled for passive side: %d\n", err); - } -#endif out: if (conn) @@ -1047,14 +1004,6 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) struct rds_ib_connect_private dp; int ret; -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled && !conn->c_loopback) { - ret = rdma_enable_apm(cm_id, RDMA_ALT_PATH_BEST); - if (ret) - printk(KERN_WARNING "RDS/IB: APM couldn't be enabled for active side: %d\n", ret); - } -#endif - rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1); ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ /* Use ic->i_flowctl as the first post credit to enable @@ -1108,11 +1057,6 @@ static void rds_ib_migrate(struct work_struct *_work) struct rdma_cm_id *cm_id = ic->i_cm_id; int ret = 0; -#if RDMA_RDS_APM_SUPPORTED - if (!rds_ib_apm_fallback) - return; -#endif - if (!ic->i_active_side) { ret = ib_query_qp(cm_id->qp, &qp_attr, IB_QP_PATH_MIG_STATE, &qp_init_attr); @@ -1140,63 +1084,6 @@ static void rds_ib_migrate(struct work_struct *_work) } } -#if RDMA_RDS_APM_SUPPORTED -void rds_ib_check_migration(struct rds_connection *conn, - struct rdma_cm_event *event) -{ - struct rds_ib_connection *ic = conn->c_transport_data; - union ib_gid sgid; - union ib_gid dgid; - struct ib_qp_init_attr qp_init_attr; - struct ib_qp_attr qp_attr; - struct rdma_cm_id *cm_id = ic->i_cm_id; - int err; - - if (!rds_ib_apm_enabled || !rds_conn_up(ic->conn)) - return ; - - ic->i_alt_path_index = event->param.ud.alt_path_index; - - memcpy(&sgid, &cm_id->route.path_rec[event->param.ud.alt_path_index]. - sgid, sizeof(union ib_gid)); - memcpy(&dgid, &cm_id->route.path_rec[event->param.ud.alt_path_index]. - dgid, sizeof(union ib_gid)); - - printk(KERN_NOTICE - "RDS/IB: connection " - "<%u.%u.%u.%u,%u.%u.%u.%u,%d> loaded alternate path " - "<"RDS_IB_GID_FMT","RDS_IB_GID_FMT">\n", - NIPQUAD(conn->c_laddr), - NIPQUAD(conn->c_faddr), - conn->c_tos, - RDS_IB_GID_ARG(sgid), RDS_IB_GID_ARG(dgid)); - - err = ib_query_qp(cm_id->qp, &qp_attr, IB_QP_ALT_PATH, &qp_init_attr); - if (err) { - printk(KERN_ERR "RDS/IB: ib_query_qp failed (%d)\n", err); - return; - } - qp_attr.alt_timeout = rds_ib_apm_timeout; - err = ib_modify_qp(cm_id->qp, &qp_attr, IB_QP_ALT_PATH); - if (err) { - printk(KERN_ERR "RDS/IB: ib_modify_qp failed (%d)\n", err); - return; - } - - if (!memcmp(&ic->i_pri_path.p_sgid, &sgid, sizeof(union ib_gid)) && - !memcmp(&ic->i_pri_path.p_dgid, &dgid, sizeof(union ib_gid))) { - if (memcmp(&ic->i_cur_path.p_sgid, &ic->i_pri_path.p_sgid, - sizeof(union ib_gid)) || - memcmp(&ic->i_cur_path.p_dgid, &ic->i_pri_path.p_dgid, - sizeof(union ib_gid))) { - - ic->i_migrate_w.ic = ic; - queue_delayed_work(rds_wq, &ic->i_migrate_w.work, 0); - } - } -} -#endif - int rds_ib_conn_connect(struct rds_connection *conn) { struct rds_ib_connection *ic = conn->c_transport_data; diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c index 384669373635..c4298c503237 100644 --- a/net/rds/ib_send.c +++ b/net/rds/ib_send.c @@ -350,8 +350,7 @@ void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc) conn->c_tos, wc->status, wc->vendor_err); rds_rtd(RDS_RTD_ERR, "status %u => %s\n", wc->status, rds_ib_wc_status_str(wc->status)); - } else - ic->i_last_migration = 0; + } } /* diff --git a/net/rds/ib_stats.c b/net/rds/ib_stats.c index 2be25a21cdc2..96c78598cccd 100644 --- a/net/rds/ib_stats.c +++ b/net/rds/ib_stats.c @@ -80,7 +80,6 @@ static char *rds_ib_stat_names[] = { "ib_srq_lows", "ib_srq_refills", "ib_srq_empty_refills", - "ib_apm_reconnect", "ib_recv_cache_added", "ib_recv_cache_removed", }; diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c index 483905d58eca..0c8e34566dc2 100644 --- a/net/rds/rdma_transport.c +++ b/net/rds/rdma_transport.c @@ -71,12 +71,6 @@ static char *rds_cm_event_strings[] = { RDS_CM_EVENT_STRING(MULTICAST_ERROR), RDS_CM_EVENT_STRING(ADDR_CHANGE), RDS_CM_EVENT_STRING(TIMEWAIT_EXIT), -#if RDMA_RDS_APM_SUPPORTED - RDS_CM_EVENT_STRING(ALT_ROUTE_RESOLVED), - RDS_CM_EVENT_STRING(ALT_ROUTE_ERROR), - RDS_CM_EVENT_STRING(LOAD_ALT_PATH), - RDS_CM_EVENT_STRING(ALT_PATH_LOADED), -#endif #undef RDS_CM_EVENT_STRING }; @@ -131,11 +125,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_ADDR_RESOLVED: rdma_set_service_type(cm_id, conn->c_tos); -#if RDMA_RDS_APM_SUPPORTED - if (rds_ib_apm_enabled) - rdma_set_timeout(cm_id, rds_ib_apm_timeout); -#endif - if (conn->c_tos && conn->c_reconnect) { struct rds_ib_connection *base_ic = conn->c_base_conn->c_transport_data; @@ -221,22 +210,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, } break; -#if RDMA_RDS_APM_SUPPORTED - case RDMA_CM_EVENT_ALT_PATH_LOADED: - rdsdebug("RDS: alt path loaded\n"); - if (conn) - trans->check_migration(conn, event); - break; - - case RDMA_CM_EVENT_ALT_ROUTE_RESOLVED: - rdsdebug("RDS: alt route resolved\n"); - break; - - case RDMA_CM_EVENT_ALT_ROUTE_ERROR: - rdsdebug("RDS: alt route resolve error\n"); - break; -#endif - case RDMA_CM_EVENT_ROUTE_ERROR: /* IP might have been moved so flush the ARP entry and retry */ page = alloc_page(GFP_HIGHUSER); @@ -339,16 +312,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, "ADDR_CHANGE event <%u.%u.%u.%u,%u.%u.%u.%u>\n", NIPQUAD(conn->c_laddr), NIPQUAD(conn->c_faddr)); -#if RDMA_RDS_APM_SUPPORTED - if (conn && !rds_ib_apm_enabled) { - rds_rtd(RDS_RTD_CM, - "ADDR_CHANGE: calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n", - NIPQUAD(conn->c_laddr), NIPQUAD(conn->c_faddr), - conn->c_tos); - conn->c_drop_source = DR_IB_ADDR_CHANGE; - rds_conn_drop(conn); - } -#else if (conn) { rds_rtd(RDS_RTD_CM, "ADDR_CHANGE: calling rds_conn_drop <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n", @@ -357,7 +320,6 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, conn->c_drop_source = DR_IB_ADDR_CHANGE; rds_conn_drop(conn); } -#endif break; case RDMA_CM_EVENT_DISCONNECTED: diff --git a/net/rds/rds.h b/net/rds/rds.h index f856c55b8aca..6978f0db8424 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -26,7 +26,6 @@ #define RDS_PROTOCOL(maj, min) (((maj) << 8) | min) #define IB_RDS_CQ_VECTOR_SUPPORTED 0 -#define RDMA_RDS_APM_SUPPORTED 0 /* * XXX randomly chosen, but at least seems to be unused: