From: Pradeep Gopanapalli Date: Thu, 10 Nov 2016 23:45:37 +0000 (+0000) Subject: xsigo: supported SGE's for LSO QP X-Git-Tag: v4.1.12-92~36^2~1 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=a9767118fb6d1a19c4f1abc298d573ca50bd754c;p=users%2Fjedix%2Flinux-maple.git xsigo: supported SGE's for LSO QP Orabug: 25029868 PSIF supports 15 SGE's in case LSO QP linearize skb if number of frags is greater than allowed sge's Reported-by: Pradeep Gopanapalli Signed-off-by: Pradeep Gopanapalli Reviewed-by: viswa krishnamurthy Reviewed-by: UmaShankar Tumari Mahabalagiri --- diff --git a/drivers/infiniband/ulp/xsigo/xscore/Makefile b/drivers/infiniband/ulp/xsigo/xscore/Makefile index d800cc89ecfe..fe29e002e3a5 100644 --- a/drivers/infiniband/ulp/xsigo/xscore/Makefile +++ b/drivers/infiniband/ulp/xsigo/xscore/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XSCORE) := xscore.o xscore-y := xscore_impl.o xs_ud.o xscore_api.o xsmp.o \ xscore_stats.o xscore_uadm.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8037\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8038\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile index 9baf33ffa148..85b14c8dc5e5 100644 --- a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile @@ -3,7 +3,7 @@ xsvhba-y := vhba_main.o vhba_xsmp.o vhba_create.o vhba_init.o vhba_delete.o \ vhba_attr.o vhba_wq.o vhba_proc.o vhba_stats.o vhba_ib.o \ vhba_scsi_intf.o vhba_align.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8037\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8038\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile index abd8202f8f8d..b5daeb1bdc38 100644 --- a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile @@ -1,7 +1,7 @@ obj-$(CONFIG_INFINIBAND_XSVNIC) := xsvnic.o xsvnic-y := xsvnic_main.o xsvnic_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8037\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8038\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/Makefile b/drivers/infiniband/ulp/xsigo/xve/Makefile index 58f03f2aad8d..892ce1385347 100644 --- a/drivers/infiniband/ulp/xsigo/xve/Makefile +++ b/drivers/infiniband/ulp/xsigo/xve/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XVE) := xve.o xve-y := xve_main.o xve_verbs.o xve_multicast.o xve_ib.o xve_tables.o \ xve_ethtool.o xve_cm.o xve_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8037\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8038\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/xve.h b/drivers/infiniband/ulp/xsigo/xve/xve.h index f0edb3b2b14c..03d8ea1c0796 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve.h +++ b/drivers/infiniband/ulp/xsigo/xve/xve.h @@ -241,6 +241,7 @@ enum { XVE_TX_WMARK_REACH_COUNTER, XVE_TX_WAKE_UP_COUNTER, XVE_TX_QUEUE_STOP_COUNTER, + XVE_TX_LIN_SKB_COUNTER, XVE_RX_SKB_COUNTER, XVE_RX_SKB_ALLOC_COUNTER, XVE_RX_SMALLSKB_ALLOC_COUNTER, @@ -774,6 +775,7 @@ struct xve_dev_priv { struct ib_send_wr tx_wr; struct ib_wc send_wc[MAX_SEND_CQE]; struct ib_recv_wr rx_wr; + uint32_t max_send_sge; /* Allocate EDR SG for now */ struct ib_sge rx_sge[XVE_UD_RX_EDR_SG]; struct ib_wc ibwc[XVE_NUM_WC]; @@ -1309,8 +1311,43 @@ static inline u16 xg_vlan_get_rxtag(struct sk_buff *skb) } + +static inline int xve_linearize_skb(struct net_device *dev, + struct sk_buff *skb, + struct xve_dev_priv *priv, + unsigned max_send_sge) +{ + unsigned usable_sge = max_send_sge - !!skb_headlen(skb); + + if (skb_shinfo(skb)->nr_frags > usable_sge) { + if (skb_linearize(skb) < 0) { + pr_warn_ratelimited("XVE: %s failure to linearize\n", + priv->xve_name); + INC_TX_DROP_STATS(priv, dev); + INC_TX_ERROR_STATS(priv, dev); + dev_kfree_skb_any(skb); + return -1; + } + + /* skb_linearize returned ok but still not reducing nr_frags */ + if (skb_shinfo(skb)->nr_frags > usable_sge) { + pr_warn_ratelimited + ("XVE: %s too many frags after skb linearize\n", + priv->xve_name); + INC_TX_DROP_STATS(priv, dev); + INC_TX_ERROR_STATS(priv, dev); + dev_kfree_skb_any(skb); + return -1; + } + priv->counters[XVE_TX_LIN_SKB_COUNTER]++; + } + return 0; + +} + + /* - * ipoib_calc_speed - calculate port speed + * xve_calc_speed - calculate port speed * * @priv - device private data * diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c index f21b5ca02cf4..16837dd02aeb 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c @@ -1438,6 +1438,9 @@ int xve_cm_dev_init(struct net_device *dev) return ret; } + /* PSIF determines SGE value based on stack unwind */ + priv->dev_attr = attr; + /* Based on the admin mtu from the chassis */ attr.max_srq_sge = min_t(int, diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c index 4cae01e27910..f08c509df08f 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c @@ -835,6 +835,10 @@ int xve_send(struct net_device *dev, struct sk_buff *skb, hlen = 0; } + /* Linearize if there are more than allowed SGE's */ + if (xve_linearize_skb(dev, skb, priv, priv->max_send_sge) < 0) + return ret; + xve_dbg_data(priv, "%s sending packet, length=%d address=%p qpn=0x%06x\n", __func__, skb->len, address, qpn); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_stats.c b/drivers/infiniband/ulp/xsigo/xve/xve_stats.c index b7497474e210..537d253c9ae4 100755 --- a/drivers/infiniband/ulp/xsigo/xve/xve_stats.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_stats.c @@ -77,6 +77,7 @@ static char *counter_name[XVE_MAX_COUNTERS] = { "tx ring wmark reached count:\t", "tx wake up count\t\t", "tx queue stop count:\t\t", + "tx linearize skb count:\t", "rx_skb_count:\t\t\t", "rx_skb_alloc_count:\t\t", "rx_smallskb_alloc_count:\t", @@ -471,6 +472,7 @@ static int xve_proc_read_device(struct seq_file *m, void *data) seq_printf(m, "IB MAX MTU: \t\t\t%d\n", vp->max_ib_mtu); seq_printf(m, "SG UD Mode:\t\t\t%d\n", xve_ud_need_sg(vp->admin_mtu)); seq_printf(m, "Max SG supported(HCA):\t\t%d\n", vp->dev_attr.max_sge); + seq_printf(m, "SG Capability:\t\t\t%d\n", vp->max_send_sge); seq_printf(m, "Eoib:\t\t\t\t%s\n", (vp->is_eoib) ? "yes" : "no"); seq_printf(m, "Jumbo:\t\t\t\t%s\n", (vp->is_jumbo) ? "yes" : "no"); seq_printf(m, "Titan:\t\t\t\t%s\n", (vp->is_titan) ? "yes" : "no"); @@ -495,7 +497,7 @@ static int xve_proc_read_device(struct seq_file *m, void *data) seq_puts(m, "Multicast Report:\n"); seq_printf(m, "Flag: \t\t%lx\n", vp->broadcast->flags); - seq_printf(m, "join state: \t\t%s\n", + seq_printf(m, "join state: \t\t%s\n", test_bit(XVE_MCAST_FLAG_ATTACHED, &vp->broadcast-> flags) ? "Joined" : "Not joined"); @@ -606,7 +608,7 @@ static int xve_proc_read_device(struct seq_file *m, void *data) if (vp->work_queue_failed != 0) seq_printf(m, "WQ Failed:\t\t\t%ld\n", vp->work_queue_failed); - seq_printf(m, "TX Net queue \t\t%s %d:%d\n", + seq_printf(m, "TX Net queue \t\t\t%s %d:%d\n", netif_queue_stopped(vp->netdev) ? "stopped" : "active", vp->counters[XVE_TX_WAKE_UP_COUNTER], vp->counters[XVE_TX_QUEUE_STOP_COUNTER]); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_verbs.c b/drivers/infiniband/ulp/xsigo/xve/xve_verbs.c index aa311508970b..2afcf7362990 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_verbs.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_verbs.c @@ -228,21 +228,22 @@ int xve_transport_dev_init(struct net_device *dev, struct ib_device *ca) if (priv->hca_caps & IB_DEVICE_BLOCK_MULTICAST_LOOPBACK) init_attr.create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; - if (dev->features & NETIF_F_SG) { - /* As Titan Card supports less than MAX SKB we need to check */ - max_sge = priv->dev_attr.max_sge; - if (max_sge >= (MAX_SKB_FRAGS + 1)) - max_sge = MAX_SKB_FRAGS + 1; - init_attr.cap.max_send_sge = max_sge; - } - if (priv->is_eoib && priv->is_titan) { init_attr.create_flags |= IB_QP_CREATE_EOIB; init_attr.cap.max_inline_data = xve_max_inline_data; - xve_debug(DEBUG_QP_INFO, priv, "Setting eoIB mode%x data%x\n", - init_attr.create_flags, xve_max_inline_data); } + if (dev->features & NETIF_F_SG) { + /*PSIF supports 16 SGE's*/ + max_sge = min_t(uint32_t, priv->dev_attr.max_sge, + MAX_SKB_FRAGS + 1); + + init_attr.cap.max_send_sge = max_sge; + } + + xve_debug(DEBUG_QP_INFO, priv, "Create QP flags%x sge%d\n", + init_attr.create_flags, max_sge); + priv->qp = ib_create_qp(priv->pd, &init_attr); if (IS_ERR(priv->qp)) { pr_warn("%s: failed to create QP\n", ca->name); @@ -270,6 +271,7 @@ int xve_transport_dev_init(struct net_device *dev, struct ib_device *ca) } priv->rx_wr.next = NULL; priv->rx_wr.sg_list = priv->rx_sge; + priv->max_send_sge = init_attr.cap.max_send_sge; return 0;