From eecd0fd51cfc32b3a7cc548f776b0583bbcb7a06 Mon Sep 17 00:00:00 2001 From: Wei Lin Guay Date: Tue, 10 Oct 2017 09:19:03 +0200 Subject: [PATCH] net/rds: set the rds_ib_init_frag based on supported sge MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The large fragment size requires the underlying HCA to support N sge. Thus, this patch sets the rds_ib_init_frag based on the minimal of (rds_ibdev->max_sge - 1) * PAGE_SIZE and the rds_ib_max_frag module_param. Orabug: 26770234 Signed-off-by: Wei Lin Guay Reviewed-by: HÃ¥kon Bugge Tested-by: Shih-Yu Huang Acked-by: Santosh Shilimkar Signed-off-by: Dhaval Giani --- net/rds/ib.c | 1 + net/rds/ib.h | 1 + net/rds/ib_cm.c | 30 ++++++++++++++++++------------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/net/rds/ib.c b/net/rds/ib.c index 9a7ecdcd56e6..a71c7e8caee0 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -2032,6 +2032,7 @@ void rds_ib_add_one(struct ib_device *device) rds_ibdev->max_wrs = dev_attr->max_qp_wr; rds_ibdev->max_sge = min(dev_attr->max_sge, RDS_IB_MAX_SGE); + WARN_ON(rds_ibdev->max_sge < 2); rds_ibdev->fmr_max_remaps = dev_attr->max_map_per_fmr?: 32; rds_ibdev->max_1m_fmrs = dev_attr->max_mr ? diff --git a/net/rds/ib.h b/net/rds/ib.h index 0c765ca6000d..9454a6dfb1b2 100644 --- a/net/rds/ib.h +++ b/net/rds/ib.h @@ -222,6 +222,7 @@ struct rds_ib_connection { u16 i_frag_cache_sz; u8 i_frag_pages; u8 i_flags; + u16 i_hca_sge; /* Batched completions */ unsigned int i_unsignaled_wrs; diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 5e476d971bf2..85fa2a978535 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -164,27 +164,30 @@ void rds_ib_init_frag(unsigned int version) } /* Update the RDS IB frag size */ -static void rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag) +static u16 rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag) { struct rds_ib_connection *ic = conn->c_transport_data; u16 current_frag = ic->i_frag_sz; u16 frag; - if (ib_init_frag_size != dp_frag) { - frag = min_t(unsigned int, dp_frag, ib_init_frag_size); + frag = min_t(unsigned int, ib_init_frag_size, + PAGE_ALIGN((ic->i_hca_sge - 1) * PAGE_SIZE)); + + if (frag != dp_frag) { + frag = min_t(unsigned int, dp_frag, frag); ic->i_frag_sz = rds_ib_get_frag(conn->c_version, frag); } else { - ic->i_frag_sz = ib_init_frag_size; + ic->i_frag_sz = frag; } - ic->i_frag_pages = ic->i_frag_sz / PAGE_SIZE; - if (!ic->i_frag_pages) - ic->i_frag_pages = 1; + ic->i_frag_pages = ceil(ic->i_frag_sz, PAGE_SIZE); pr_debug("RDS/IB: conn <%pI4, %pI4,%d>, Frags : {%d,%d,%d}, updated {%d -> %d}\n", &conn->c_laddr, &conn->c_faddr, conn->c_tos, ib_init_frag_size / SZ_1K, ic->i_frag_sz / SZ_1K, dp_frag / SZ_1K, current_frag / SZ_1K, ic->i_frag_sz / SZ_1K); + + return ic->i_frag_sz; } /* Init per IC frag size */ @@ -762,6 +765,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn) attr.srq = rds_ibdev->srq->s_srq; } + ic->i_hca_sge = rds_ibdev->max_sge; /* * XXX this can fail if max_*_wr is too large? Are we supposed * to back off until we get a value that the hardware can support? @@ -892,6 +896,7 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, u32 version; int err = 1, destroy = 1; int acl_ret = 0; + u16 frag; /* Check whether the remote protocol version matches ours. */ version = rds_ib_protocol_compatible(event); @@ -925,7 +930,6 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, } rds_ib_set_protocol(conn, version); - rds_ib_set_frag_size(conn, be16_to_cpu(dp->dp_frag_sz)); conn->c_acl_en = acl_ret; conn->c_acl_init = 1; @@ -1017,11 +1021,12 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, rds_conn_drop(conn, DR_IB_PAS_SETUP_QP_FAIL); goto out; } + frag = rds_ib_set_frag_size(conn, be16_to_cpu(dp->dp_frag_sz)); rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version, event->param.conn.responder_resources, event->param.conn.initiator_depth, - ib_init_frag_size); + frag); /* rdma_accept() calls rdma_reject() internally if it fails */ err = rdma_accept(cm_id, &conn_param); @@ -1071,6 +1076,7 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) struct rds_ib_connection *ic = conn->c_transport_data; struct rdma_conn_param conn_param; struct rds_ib_connect_private dp; + u16 frag; int ret; ret = rds_ib_match_acl(ic->i_cm_id, conn->c_faddr); @@ -1104,10 +1110,10 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) rds_conn_drop(conn, DR_IB_ACT_SETUP_QP_FAIL); goto out; } - + frag = rds_ib_set_frag_size(conn, ib_init_frag_size); rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, - conn->c_proposed_version, UINT_MAX, UINT_MAX, - ib_init_frag_size); + conn->c_proposed_version, UINT_MAX, UINT_MAX, + frag); ret = rdma_connect(cm_id, &conn_param); if (ret) { pr_warn("RDS/IB: rdma_connect failed (%d)\n", ret); -- 2.50.1