net/rds: set the rds_ib_init_frag based on supported sge

The large fragment size requires the underlying HCA to support N sge. Thus,
this patch sets the rds_ib_init_frag based on the minimal of
(rds_ibdev->max_sge - 1) * PAGE_SIZE and the rds_ib_max_frag module_param.

Orabug: 26770234

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>
Tested-by: Shih-Yu Huang <shih-yu.huang@oracle.com>
Acked-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>
Signed-off-by: Dhaval Giani <dhaval.giani@oracle.com>

diff --git a/net/rds/ib.c b/net/rds/ib.c
index 9a7ecdcd56e60af79f84a3ba6234b44bec9611dc..a71c7e8caee0469b65c675a5a0981abdc153d237 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -2032,6 +2032,7 @@ void rds_ib_add_one(struct ib_device *device)
        rds_ibdev->max_wrs = dev_attr->max_qp_wr;
        rds_ibdev->max_sge = min(dev_attr->max_sge, RDS_IB_MAX_SGE);
 
+       WARN_ON(rds_ibdev->max_sge < 2);
        rds_ibdev->fmr_max_remaps = dev_attr->max_map_per_fmr?: 32;
 
        rds_ibdev->max_1m_fmrs = dev_attr->max_mr ?

diff --git a/net/rds/ib.h b/net/rds/ib.h
index 0c765ca6000d3a98e46611d4a3cab3e7bd202707..9454a6dfb1b2f2720b04fd67bf92a774b4c496f0 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -222,6 +222,7 @@ struct rds_ib_connection {
        u16                     i_frag_cache_sz;
        u8                      i_frag_pages;
        u8                      i_flags;
+       u16                     i_hca_sge;
 
        /* Batched completions */
        unsigned int            i_unsignaled_wrs;

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 5e476d971bf23496adc7f8574e9bb82596d9a07f..85fa2a978535841ceb52e49111ba9da9d43d07fe 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -164,27 +164,30 @@ void rds_ib_init_frag(unsigned int version)
 }
 
 /* Update the RDS IB frag size */
-static void rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag)
+static u16 rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag)
 {
        struct rds_ib_connection *ic = conn->c_transport_data;
        u16 current_frag = ic->i_frag_sz;
        u16 frag;
 
-       if (ib_init_frag_size != dp_frag) {
-               frag = min_t(unsigned int, dp_frag, ib_init_frag_size);
+       frag = min_t(unsigned int, ib_init_frag_size,
+                    PAGE_ALIGN((ic->i_hca_sge - 1) * PAGE_SIZE));
+
+       if (frag != dp_frag) {
+               frag = min_t(unsigned int, dp_frag, frag);
                ic->i_frag_sz = rds_ib_get_frag(conn->c_version, frag);
        } else {
-               ic->i_frag_sz = ib_init_frag_size;
+               ic->i_frag_sz = frag;
        }
 
-       ic->i_frag_pages =  ic->i_frag_sz / PAGE_SIZE;
-       if (!ic->i_frag_pages)
-               ic->i_frag_pages = 1;
+       ic->i_frag_pages =  ceil(ic->i_frag_sz, PAGE_SIZE);
 
        pr_debug("RDS/IB: conn <%pI4, %pI4,%d>, Frags <init,ic,dp>: {%d,%d,%d}, updated {%d -> %d}\n",
                 &conn->c_laddr, &conn->c_faddr, conn->c_tos,
                 ib_init_frag_size / SZ_1K, ic->i_frag_sz / SZ_1K, dp_frag /  SZ_1K,
                 current_frag / SZ_1K, ic->i_frag_sz / SZ_1K);
+
+       return ic->i_frag_sz;
 }
 
 /* Init per IC frag size */
@@ -762,6 +765,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
                attr.srq = rds_ibdev->srq->s_srq;
        }
 
+       ic->i_hca_sge = rds_ibdev->max_sge;
        /*
         * XXX this can fail if max_*_wr is too large?  Are we supposed
         * to back off until we get a value that the hardware can support?
@@ -892,6 +896,7 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
        u32 version;
        int err = 1, destroy = 1;
        int acl_ret = 0;
+       u16 frag;
 
        /* Check whether the remote protocol version matches ours. */
        version = rds_ib_protocol_compatible(event);
@@ -925,7 +930,6 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
        }
 
        rds_ib_set_protocol(conn, version);
-       rds_ib_set_frag_size(conn, be16_to_cpu(dp->dp_frag_sz));
 
        conn->c_acl_en = acl_ret;
        conn->c_acl_init = 1;
@@ -1017,11 +1021,12 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
                rds_conn_drop(conn, DR_IB_PAS_SETUP_QP_FAIL);
                goto out;
        }
+       frag = rds_ib_set_frag_size(conn, be16_to_cpu(dp->dp_frag_sz));
 
        rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
                event->param.conn.responder_resources,
                event->param.conn.initiator_depth,
-               ib_init_frag_size);
+               frag);
 
        /* rdma_accept() calls rdma_reject() internally if it fails */
        err = rdma_accept(cm_id, &conn_param);
@@ -1071,6 +1076,7 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
        struct rds_ib_connection *ic = conn->c_transport_data;
        struct rdma_conn_param conn_param;
        struct rds_ib_connect_private dp;
+       u16 frag;
        int ret;
 
        ret = rds_ib_match_acl(ic->i_cm_id, conn->c_faddr);
@@ -1104,10 +1110,10 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
                rds_conn_drop(conn, DR_IB_ACT_SETUP_QP_FAIL);
                goto out;
        }
-
+       frag = rds_ib_set_frag_size(conn, ib_init_frag_size);
        rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
-                               conn->c_proposed_version, UINT_MAX, UINT_MAX,
-                               ib_init_frag_size);
+                                 conn->c_proposed_version, UINT_MAX, UINT_MAX,
+                                 frag);
        ret = rdma_connect(cm_id, &conn_param);
        if (ret) {
                pr_warn("RDS/IB: rdma_connect failed (%d)\n", ret);