From: Pradeep Gopanapalli Date: Tue, 1 Nov 2016 19:41:13 +0000 (+0000) Subject: xsigo: hard LOCKUP in freeing paths X-Git-Tag: v4.1.12-92~45^2~2 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=2aa36a8278cc9314cafbb413a53f0d79aadc9310;p=users%2Fjedix%2Flinux-maple.git xsigo: hard LOCKUP in freeing paths Orabug: 24669507 When path->users becomes zero uVNIC driver starts cleaning up the Forwarding table entries. In some corner cases the call is invoked from transmit function which is in interrupt context and that results in a hard LOCKUP. With new changes path->users is decremented in transmit function to allow cleanup to happen from other thread. Proper care is taken to avoid race between these two contexts. Reported-by: chien yen Signed-off-by: Pradeep Gopanapalli Reviewed-by: Aravind Kini Reviewed-by: viswa krishnamurthy Reviewed-by: Manish Kumar Singh Reviewed-by: UmaShankar Tumari Mahabalagiri --- diff --git a/drivers/infiniband/ulp/xsigo/xscore/Makefile b/drivers/infiniband/ulp/xsigo/xscore/Makefile index 06e05f91bd648..4d6ee34ba069b 100644 --- a/drivers/infiniband/ulp/xsigo/xscore/Makefile +++ b/drivers/infiniband/ulp/xsigo/xscore/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XSCORE) := xscore.o xscore-y := xscore_impl.o xs_ud.o xscore_api.o xsmp.o \ xscore_stats.o xscore_uadm.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile index 08e1288516701..5d562d1ac5ef5 100644 --- a/drivers/infiniband/ulp/xsigo/xsvhba/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvhba/Makefile @@ -3,7 +3,7 @@ xsvhba-y := vhba_main.o vhba_xsmp.o vhba_create.o vhba_init.o vhba_delete.o \ vhba_attr.o vhba_wq.o vhba_proc.o vhba_stats.o vhba_ib.o \ vhba_scsi_intf.o vhba_align.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile index 3f4edcdcc46cc..7d856ea2d029b 100644 --- a/drivers/infiniband/ulp/xsigo/xsvnic/Makefile +++ b/drivers/infiniband/ulp/xsigo/xsvnic/Makefile @@ -1,7 +1,7 @@ obj-$(CONFIG_INFINIBAND_XSVNIC) := xsvnic.o xsvnic-y := xsvnic_main.o xsvnic_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/Makefile b/drivers/infiniband/ulp/xsigo/xve/Makefile index 1f038eabdbc4f..cfcef1d931198 100644 --- a/drivers/infiniband/ulp/xsigo/xve/Makefile +++ b/drivers/infiniband/ulp/xsigo/xve/Makefile @@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XVE) := xve.o xve-y := xve_main.o xve_verbs.o xve_multicast.o xve_ib.o xve_tables.o \ xve_ethtool.o xve_cm.o xve_stats.o -ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8031\" +ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8033\" ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c index d3853d039e006..78c83ea9ef79c 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_cm.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_cm.c @@ -1224,7 +1224,6 @@ void xve_cm_destroy_tx_deferred(struct xve_cm_ctx *tx) clear_bit(XVE_FLAG_OPER_UP, &tx->flags); if (test_and_clear_bit(XVE_FLAG_INITIALIZED, &tx->flags)) { list_move(&tx->list, &priv->cm.reap_list); - xve_cm_set(tx->path, NULL); xve_queue_work(priv, XVE_WQ_START_CMTXREAP); } spin_unlock_irqrestore(&priv->lock, flags); @@ -1282,6 +1281,7 @@ static void __xve_cm_tx_reap(struct xve_dev_priv *priv) if (p->path) xve_flush_single_path_by_gid(dev, &p->path->pathrec.dgid); + xve_cm_set(p->path, NULL); xve_cm_tx_destroy(p); netif_tx_lock_bh(dev); spin_lock_irqsave(&priv->lock, flags); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c index 08c3850a20ff2..09b822cc23da2 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_ib.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_ib.c @@ -814,7 +814,7 @@ int xve_send(struct net_device *dev, struct sk_buff *skb, if (unlikely(skb->len > max_packet_len)) { xve_warn(priv, "%s packet len %d", __func__, skb->len); - xve_warn(priv, "(> %d) too long to", priv->mcast_mtu); + xve_warn(priv, "(> %d) too long to", max_packet_len); xve_warn(priv, "send,dropping %ld packets %s\n", dev->stats.tx_dropped, dev->name); INC_TX_DROP_STATS(priv, dev); diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_main.c b/drivers/infiniband/ulp/xsigo/xve/xve_main.c index 67a96087b4514..dad67fff015cb 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_main.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_main.c @@ -341,39 +341,42 @@ inline void xve_get_path(struct xve_path *path) atomic_inc(&path->users); } -inline void xve_put_path(struct xve_path *path, int do_lock) +inline void xve_put_path(struct xve_path *path) +{ + atomic_dec_if_positive(&path->users); +} + +inline void xve_free_path(struct xve_path *path, int do_lock) { struct xve_dev_priv *priv; struct net_device *netdev; struct sk_buff *skb; unsigned long flags = 0; - if (atomic_dec_and_test(&path->users)) { - netdev = path->dev; - priv = netdev_priv(netdev); - while ((skb = __skb_dequeue(&path->queue))) - dev_kfree_skb_irq(skb); + netdev = path->dev; + priv = netdev_priv(netdev); + while ((skb = __skb_dequeue(&path->queue))) + dev_kfree_skb_irq(skb); - while ((skb = __skb_dequeue(&path->uplink_queue))) - dev_kfree_skb_irq(skb); + while ((skb = __skb_dequeue(&path->uplink_queue))) + dev_kfree_skb_irq(skb); - if (do_lock) - spin_lock_irqsave(&priv->lock, flags); - if (xve_cmtx_get(path)) { - if (do_lock) - spin_unlock_irqrestore(&priv->lock, flags); - xve_cm_destroy_tx_deferred(xve_cmtx_get(path)); - if (do_lock) - spin_lock_irqsave(&priv->lock, flags); - } - xve_flush_l2_entries(netdev, path); - if (path->ah) - xve_put_ah(path->ah); + if (do_lock) + spin_lock_irqsave(&priv->lock, flags); + if (xve_cmtx_get(path)) { if (do_lock) spin_unlock_irqrestore(&priv->lock, flags); - - kfree(path); + xve_cm_destroy_tx_deferred(xve_cmtx_get(path)); + if (do_lock) + spin_lock_irqsave(&priv->lock, flags); } + xve_flush_l2_entries(netdev, path); + if (path->ah) + xve_put_ah(path->ah); + if (do_lock) + spin_unlock_irqrestore(&priv->lock, flags); + + kfree(path); } struct xve_path *__path_find(struct net_device *netdev, void *gid) @@ -529,7 +532,28 @@ void xve_flush_single_path_by_gid(struct net_device *dev, union ib_gid *gid) wait_for_completion(&path->done); list_del(&path->list); - xve_put_path(path, 1); + /* Make sure path is not in use */ + if (atomic_dec_if_positive(&path->users) <= 0) + xve_free_path(path, 1); + else { + /* Wait for path->users to become zero */ + unsigned long begin = jiffies; + + while (atomic_read(&path->users)) { + if (time_after(jiffies, begin + 5 * HZ)) { + xve_warn(priv, "%p Waited to free path %pI6", + path, path->pathrec.dgid.raw); + goto timeout; + } + msleep(20); + } + if (atomic_read(&path->users) == 0) + xve_free_path(path, 1); + + } +timeout: + return; + } void xve_flush_single_path(struct net_device *dev, struct xve_path *path) @@ -763,7 +787,7 @@ xve_path_lookup(struct net_device *dev, spin_unlock_irqrestore(&xve_fwt->lock, flags); if (!path->ah) { if (!path->query && path_rec_start(dev, path)) { - xve_put_path(path, 0); + xve_free_path(path, 0); return NULL; } } @@ -821,7 +845,7 @@ int xve_gw_send(struct net_device *dev, struct sk_buff *skb) priv->counters[XVE_GW_MCAST_TX]++; out: - xve_put_path(path, 0); + xve_put_path(path); return ret; } @@ -837,7 +861,7 @@ int xve_add_eoib_header(struct xve_dev_priv *priv, struct sk_buff *skb) if (!skb_new) return -1; - kfree_skb(skb); + dev_kfree_skb_any(skb); skb = skb_new; } eoibp = (struct xve_eoib_hdr *) skb_push(skb, len); @@ -854,7 +878,7 @@ static int xve_start_xmit(struct sk_buff *skb, struct net_device *dev) struct xve_dev_priv *priv = netdev_priv(dev); struct xve_fwt_entry *fwt_entry = NULL; struct xve_path *path = NULL; - unsigned long flags; + unsigned long flags = 0; int ret = NETDEV_TX_OK, len = 0; u8 skb_need_tofree = 0, inc_drop_cnt = 0, queued_pkt = 0; u16 vlan_tag = 0; @@ -1009,7 +1033,7 @@ stats: INC_TX_BYTE_STATS(priv, dev, len); free_fwt_ctx: if (path) - xve_put_path(path, 0); + xve_put_path(path); xve_fwt_put_ctx(&priv->xve_fwt, fwt_entry); unlock: if (inc_drop_cnt) @@ -1018,7 +1042,7 @@ unlock: if (!queued_pkt) dev->trans_start = jiffies; if (skb_need_tofree) - dev_kfree_skb(skb); + dev_kfree_skb_any(skb); spin_unlock_irqrestore(&priv->lock, flags); return ret; diff --git a/drivers/infiniband/ulp/xsigo/xve/xve_tables.c b/drivers/infiniband/ulp/xsigo/xve/xve_tables.c index ee0ce70678d7d..bf6a1eed4f8ef 100644 --- a/drivers/infiniband/ulp/xsigo/xve/xve_tables.c +++ b/drivers/infiniband/ulp/xsigo/xve/xve_tables.c @@ -326,7 +326,7 @@ void xve_remove_fwt_entry(struct xve_dev_priv *priv, struct xve_fwt_entry *fwt_entry) { struct xve_fwt_s *xve_fwt = &priv->xve_fwt; - unsigned long flags; + unsigned long flags = 0; spin_lock_irqsave(&xve_fwt->lock, flags); xve_debug(DEBUG_TABLE_INFO, priv, "%s Deleting FWT From list %p\n", @@ -366,6 +366,7 @@ void xve_fwt_entry_destroy(struct xve_dev_priv *priv, struct xve_fwt_entry *fwt_entry) { xve_remove_fwt_entry(priv, fwt_entry); + /* Function gets cald with Lock held always */ xve_fwt_entry_free(priv, fwt_entry); }