From 2aa36a8278cc9314cafbb413a53f0d79aadc9310 Mon Sep 17 00:00:00 2001 From: Pradeep Gopanapalli Date: Tue, 1 Nov 2016 19:41:13 +0000 Subject: [PATCH] xsigo: hard LOCKUP in freeing paths Orabug: 24669507 When path->users becomes zero uVNIC driver starts cleaning up the Forwarding table entries. In some corner cases the call is invoked from transmit function which is in interrupt context and that results in a hard LOCKUP. With new changes path->users is decremented in transmit function to allow cleanup to happen from other thread. Proper care is taken to avoid race between these two contexts. Reported-by: chien yen Signed-off-by: Pradeep Gopanapalli Reviewed-by: Aravind Kini Reviewed-by: viswa krishnamurthy Reviewed-by: Manish Kumar Singh Reviewed-by: UmaShankar Tumari Mahabalagiri --- drivers/infiniband/ulp/xsigo/xscore/Makefile | 2 +- drivers/infiniband/ulp/xsigo/xsvhba/Makefile | 2 +- drivers/infiniband/ulp/xsigo/xsvnic/Makefile | 2 +- drivers/infiniband/ulp/xsigo/xve/Makefile | 2 +- drivers/infiniband/ulp/xsigo/xve/xve_cm.c | 2 +- drivers/infiniband/ulp/xsigo/xve/xve_ib.c | 2 +- drivers/infiniband/ulp/xsigo/xve/xve_main.c | 82 ++++++++++++------- drivers/infiniband/ulp/xsigo/xve/xve_tables.c | 3 +- 8 files changed, 61 insertions(+), 36 deletions(-) diff --git a/drivers/infiniband/ulp/xsigo/xscore/Makefile b/drivers/infiniband/ulp/xsigo/xscore/Makefile index 06e05f91bd64..4d6ee34ba069 100644 --- a/drivers/infiniband/ulp/xsigo/xscore/Makefile +++ b/drivers/infiniband/ulp/xsigo/xscore/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XSCORE) := xscore.o xscore-y := xscore_impl.o xs_ud.o xscore_api.o xsmp.o \ xscore_stats.o xscore_uadm.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile index 08e128851670..5d562d1ac5ef 100644 --- a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile @@ -3,7 +3,7 @@ xsvhba-y := vhba_main.o vhba_xsmp.o vhba_create.o vhba_init.o vhba_delete.o \ vhba_attr.o vhba_wq.o vhba_proc.o vhba_stats.o vhba_ib.o \ vhba_scsi_intf.o vhba_align.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile index 3f4edcdcc46c..7d856ea2d029 100644 --- a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile @@ -1,7 +1,7 @@ obj-$(CONFIG_INFINIBAND_XSVNIC) := xsvnic.o xsvnic-y := xsvnic_main.o xsvnic_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/Makefile b/drivers/infiniband/ulp/xsigo/xve/Makefile index 1f038eabdbc4..cfcef1d93119 100644 --- a/drivers/infiniband/ulp/xsigo/xve/Makefile +++ b/drivers/infiniband/ulp/xsigo/xve/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XVE) := xve.o xve-y := xve_main.o xve_verbs.o xve_multicast.o xve_ib.o xve_tables.o \ xve_ethtool.o xve_cm.o xve_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c index d3853d039e00..78c83ea9ef79 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c @@ -1224,7 +1224,6 @@ void xve_cm_destroy_tx_deferred(struct xve_cm_ctx *tx) clear_bit(XVE_FLAG_OPER_UP, &tx->flags); if (test_and_clear_bit(XVE_FLAG_INITIALIZED, &tx->flags)) { list_move(&tx->list, &priv->cm.reap_list); - xve_cm_set(tx->path, NULL); xve_queue_work(priv, XVE_WQ_START_CMTXREAP); } spin_unlock_irqrestore(&priv->lock, flags); @@ -1282,6 +1281,7 @@ static void __xve_cm_tx_reap(struct xve_dev_priv *priv) if (p->path) xve_flush_single_path_by_gid(dev, &p->path->pathrec.dgid); + xve_cm_set(p->path, NULL); xve_cm_tx_destroy(p); netif_tx_lock_bh(dev); spin_lock_irqsave(&priv->lock, flags); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c index 08c3850a20ff..09b822cc23da 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c @@ -814,7 +814,7 @@ int xve_send(struct net_device *dev, struct sk_buff *skb, if (unlikely(skb->len > max_packet_len)) { xve_warn(priv, "%s packet len %d", __func__, skb->len); - xve_warn(priv, "(> %d) too long to", priv->mcast_mtu); + xve_warn(priv, "(> %d) too long to", max_packet_len); xve_warn(priv, "send,dropping %ld packets %s\n", dev->stats.tx_dropped, dev->name); INC_TX_DROP_STATS(priv, dev); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_main.c b/drivers/infiniband/ulp/xsigo/xve/xve_main.c index 67a96087b451..dad67fff015c 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_main.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_main.c @@ -341,39 +341,42 @@ inline void xve_get_path(struct xve_path *path) atomic_inc(&path->users); } -inline void xve_put_path(struct xve_path *path, int do_lock) +inline void xve_put_path(struct xve_path *path) +{ + atomic_dec_if_positive(&path->users); +} + +inline void xve_free_path(struct xve_path *path, int do_lock) { struct xve_dev_priv *priv; struct net_device *netdev; struct sk_buff *skb; unsigned long flags = 0; - if (atomic_dec_and_test(&path->users)) { - netdev = path->dev; - priv = netdev_priv(netdev); - while ((skb = __skb_dequeue(&path->queue))) - dev_kfree_skb_irq(skb); + netdev = path->dev; + priv = netdev_priv(netdev); + while ((skb = __skb_dequeue(&path->queue))) + dev_kfree_skb_irq(skb); - while ((skb = __skb_dequeue(&path->uplink_queue))) - dev_kfree_skb_irq(skb); + while ((skb = __skb_dequeue(&path->uplink_queue))) + dev_kfree_skb_irq(skb); - if (do_lock) - spin_lock_irqsave(&priv->lock, flags); - if (xve_cmtx_get(path)) { - if (do_lock) - spin_unlock_irqrestore(&priv->lock, flags); - xve_cm_destroy_tx_deferred(xve_cmtx_get(path)); - if (do_lock) - spin_lock_irqsave(&priv->lock, flags); - } - xve_flush_l2_entries(netdev, path); - if (path->ah) - xve_put_ah(path->ah); + if (do_lock) + spin_lock_irqsave(&priv->lock, flags); + if (xve_cmtx_get(path)) { if (do_lock) spin_unlock_irqrestore(&priv->lock, flags); - - kfree(path); + xve_cm_destroy_tx_deferred(xve_cmtx_get(path)); + if (do_lock) + spin_lock_irqsave(&priv->lock, flags); } + xve_flush_l2_entries(netdev, path); + if (path->ah) + xve_put_ah(path->ah); + if (do_lock) + spin_unlock_irqrestore(&priv->lock, flags); + + kfree(path); } struct xve_path *__path_find(struct net_device *netdev, void *gid) @@ -529,7 +532,28 @@ void xve_flush_single_path_by_gid(struct net_device *dev, union ib_gid *gid) wait_for_completion(&path->done); list_del(&path->list); - xve_put_path(path, 1); + /* Make sure path is not in use */ + if (atomic_dec_if_positive(&path->users) <= 0) + xve_free_path(path, 1); + else { + /* Wait for path->users to become zero */ + unsigned long begin = jiffies; + + while (atomic_read(&path->users)) { + if (time_after(jiffies, begin + 5 * HZ)) { + xve_warn(priv, "%p Waited to free path %pI6", + path, path->pathrec.dgid.raw); + goto timeout; + } + msleep(20); + } + if (atomic_read(&path->users) == 0) + xve_free_path(path, 1); + + } +timeout: + return; + } void xve_flush_single_path(struct net_device *dev, struct xve_path *path) @@ -763,7 +787,7 @@ xve_path_lookup(struct net_device *dev, spin_unlock_irqrestore(&xve_fwt->lock, flags); if (!path->ah) { if (!path->query && path_rec_start(dev, path)) { - xve_put_path(path, 0); + xve_free_path(path, 0); return NULL; } } @@ -821,7 +845,7 @@ int xve_gw_send(struct net_device *dev, struct sk_buff *skb) priv->counters[XVE_GW_MCAST_TX]++; out: - xve_put_path(path, 0); + xve_put_path(path); return ret; } @@ -837,7 +861,7 @@ int xve_add_eoib_header(struct xve_dev_priv *priv, struct sk_buff *skb) if (!skb_new) return -1; - kfree_skb(skb); + dev_kfree_skb_any(skb); skb = skb_new; } eoibp = (struct xve_eoib_hdr *) skb_push(skb, len); @@ -854,7 +878,7 @@ static int xve_start_xmit(struct sk_buff *skb, struct net_device *dev) struct xve_dev_priv *priv = netdev_priv(dev); struct xve_fwt_entry *fwt_entry = NULL; struct xve_path *path = NULL; - unsigned long flags; + unsigned long flags = 0; int ret = NETDEV_TX_OK, len = 0; u8 skb_need_tofree = 0, inc_drop_cnt = 0, queued_pkt = 0; u16 vlan_tag = 0; @@ -1009,7 +1033,7 @@ stats: INC_TX_BYTE_STATS(priv, dev, len); free_fwt_ctx: if (path) - xve_put_path(path, 0); + xve_put_path(path); xve_fwt_put_ctx(&priv->xve_fwt, fwt_entry); unlock: if (inc_drop_cnt) @@ -1018,7 +1042,7 @@ unlock: if (!queued_pkt) dev->trans_start = jiffies; if (skb_need_tofree) - dev_kfree_skb(skb); + dev_kfree_skb_any(skb); spin_unlock_irqrestore(&priv->lock, flags); return ret; diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_tables.c b/drivers/infiniband/ulp/xsigo/xve/xve_tables.c index ee0ce70678d7..bf6a1eed4f8e 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_tables.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_tables.c @@ -326,7 +326,7 @@ void xve_remove_fwt_entry(struct xve_dev_priv *priv, struct xve_fwt_entry *fwt_entry) { struct xve_fwt_s *xve_fwt = &priv->xve_fwt; - unsigned long flags; + unsigned long flags = 0; spin_lock_irqsave(&xve_fwt->lock, flags); xve_debug(DEBUG_TABLE_INFO, priv, "%s Deleting FWT From list %p\n", @@ -366,6 +366,7 @@ void xve_fwt_entry_destroy(struct xve_dev_priv *priv, struct xve_fwt_entry *fwt_entry) { xve_remove_fwt_entry(priv, fwt_entry); + /* Function gets cald with Lock held always */ xve_fwt_entry_free(priv, fwt_entry); } -- 2.50.1