xprtrdma: Prevent dereferencing r_xprt->rx_ep after it is freed

r_xprt->rx_ep is known to be good while the transport's send lock is
held.  Otherwise additional references on rx_ep must be held when it
is used outside of that lock's critical sections.

For now, bump the rx_ep reference count once whenever there is at
least one outstanding Receive WR. This avoids the memory bandwidth
overhead of taking and releasing the reference count for every
ib_post_recv() and Receive completion.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 2ae348377806ab645867f5557d1f79039a5e5620..b021baa4b28df00ef716281ac769603da1ac5edb 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -84,7 +84,8 @@ static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep);
 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
-static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep);
+static void rpcrdma_ep_get(struct rpcrdma_ep *ep);
+static int rpcrdma_ep_put(struct rpcrdma_ep *ep);
 static struct rpcrdma_regbuf *
 rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
                     gfp_t flags);
@@ -97,7 +98,8 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
  */
 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
 {
-       struct rdma_cm_id *id = r_xprt->rx_ep->re_id;
+       struct rpcrdma_ep *ep = r_xprt->rx_ep;
+       struct rdma_cm_id *id = ep->re_id;
 
        /* Flush Receives, then wait for deferred Reply work
         * to complete.
@@ -108,6 +110,8 @@ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
         * local invalidations.
         */
        ib_drain_sq(id->qp);
+
+       rpcrdma_ep_put(ep);
 }
 
 /**
@@ -266,7 +270,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
                xprt_force_disconnect(xprt);
                goto disconnected;
        case RDMA_CM_EVENT_ESTABLISHED:
-               kref_get(&ep->re_kref);
+               rpcrdma_ep_get(ep);
                ep->re_connect_status = 1;
                rpcrdma_update_cm_private(ep, &event->param.conn);
                trace_xprtrdma_inline_thresh(ep);
@@ -289,7 +293,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
                ep->re_connect_status = -ECONNABORTED;
 disconnected:
                xprt_force_disconnect(xprt);
-               return rpcrdma_ep_destroy(ep);
+               return rpcrdma_ep_put(ep);
        default:
                break;
        }
@@ -345,7 +349,7 @@ out:
        return ERR_PTR(rc);
 }
 
-static void rpcrdma_ep_put(struct kref *kref)
+static void rpcrdma_ep_destroy(struct kref *kref)
 {
        struct rpcrdma_ep *ep = container_of(kref, struct rpcrdma_ep, re_kref);
 
@@ -369,13 +373,18 @@ static void rpcrdma_ep_put(struct kref *kref)
        module_put(THIS_MODULE);
 }
 
+static noinline void rpcrdma_ep_get(struct rpcrdma_ep *ep)
+{
+       kref_get(&ep->re_kref);
+}
+
 /* Returns:
  *     %0 if @ep still has a positive kref count, or
  *     %1 if @ep was destroyed successfully.
  */
-static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep)
+static noinline int rpcrdma_ep_put(struct rpcrdma_ep *ep)
 {
-       return kref_put(&ep->re_kref, rpcrdma_ep_put);
+       return kref_put(&ep->re_kref, rpcrdma_ep_destroy);
 }
 
 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
@@ -492,7 +501,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
        return 0;
 
 out_destroy:
-       rpcrdma_ep_destroy(ep);
+       rpcrdma_ep_put(ep);
        rdma_destroy_id(id);
 out_free:
        kfree(ep);
@@ -521,8 +530,12 @@ retry:
 
        ep->re_connect_status = 0;
        xprt_clear_connected(xprt);
-
        rpcrdma_reset_cwnd(r_xprt);
+
+       /* Bump the ep's reference count while there are
+        * outstanding Receives.
+        */
+       rpcrdma_ep_get(ep);
        rpcrdma_post_recvs(r_xprt, true);
 
        rc = rpcrdma_sendctxs_create(r_xprt);
@@ -587,7 +600,7 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
        rpcrdma_mrs_destroy(r_xprt);
        rpcrdma_sendctxs_destroy(r_xprt);
 
-       if (rpcrdma_ep_destroy(ep))
+       if (rpcrdma_ep_put(ep))
                rdma_destroy_id(id);
 
        r_xprt->rx_ep = NULL;