From d1b5d340c88c6d88197f2d73f561305197f467cc Mon Sep 17 00:00:00 2001 From: Erez Shitrit Date: Sun, 8 Dec 2013 13:39:35 +0200 Subject: [PATCH] rdma_cm/cma: Cache broadcast domain record. Currently, rdma_cm waits for the IPoIB driver to complete its join to the broadcast domain record; after IPoIB gets its multicast, rdma_cm tries to obtain its own multicast. After an IB_CLIENT reregister event, IPoIB may not succeed in its first effort to reregister its multicast groups. In this case, the backoff mechanism is applied, and IPoIB retries after a backoff which starts at 2 seconds and can increase up to 16 seconds. Since rdma_cm waits for the IPoIB multicast join to succeed, it too will be delayed at least 2 seconds. The fix is to detach rdma_cm's multicast operation from IPoIB's broadcast record re-join. When rdma_cm executes a new join request, it now tries (via the cma) to take parameters from a cached broadcast record. If the join fails using the cached values, the cma deletes the cached record and tries to get a new one. Signed-off-by: Erez Shitrit (Ported from Mellanox OFED 2.4) Signed-off-by: Mukesh Kacker --- drivers/infiniband/core/cma.c | 36 +++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 6a36cdfedeca..c8db3951942f 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -180,6 +180,9 @@ struct rdma_id_private { u8 tos; u8 reuseaddr; u8 afonly; + /* cache for mc record params */ + struct ib_sa_mcmember_rec rec; + int is_valid_rec; }; struct cma_multicast { @@ -3222,9 +3225,17 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) &event.param.ud.ah_attr); event.param.ud.qp_num = 0xFFFFFF; event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); - } else + } else { event.event = RDMA_CM_EVENT_MULTICAST_ERROR; + /* mark that the cached record is no longer valid */ + if (status != -ENETRESET && status != -EAGAIN) { + spin_lock(&id_priv->lock); + id_priv->is_valid_rec = 0; + spin_unlock(&id_priv->lock); + } + } + ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { cma_exch(id_priv, RDMA_CM_DESTROYING); @@ -3273,13 +3284,26 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv, struct ib_sa_mcmember_rec rec; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; ib_sa_comp_mask comp_mask; - int ret; + int ret = 0; - ib_addr_get_mgid(dev_addr, &rec.mgid); - ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, - &rec.mgid, &rec); - if (ret) + ib_addr_get_mgid(dev_addr, &id_priv->rec.mgid); + + /* cache ipoib bc record */ + spin_lock(&id_priv->lock); + if (!id_priv->is_valid_rec) + ret = ib_sa_get_mcmember_rec(id_priv->id.device, + id_priv->id.port_num, + &id_priv->rec.mgid, + &id_priv->rec); + if (ret) { + id_priv->is_valid_rec = 0; + spin_unlock(&id_priv->lock); return ret; + } else { + rec = id_priv->rec; + id_priv->is_valid_rec = 1; + } + spin_unlock(&id_priv->lock); ret = cma_set_qkey(id_priv, 0); if (ret) -- 2.50.1