]> www.infradead.org Git - users/jedix/linux-maple.git/commitdiff
SDP: do gracefull close instead of always doing abortive close.
authorAmir Vadai <amirv@mellanox.co.il>
Thu, 10 Jul 2008 08:53:53 +0000 (01:53 -0700)
committerMukesh Kacker <mukesh.kacker@oracle.com>
Tue, 6 Oct 2015 12:04:16 +0000 (05:04 -0700)
Main changes:
1.  when a close/shutdown syscall is called, instead of sending a DREQ, put
    last socket ref count and go to TCP_CLOSE state do:
    - take a socket reference count
    - set state to TCP_TIME_WAIT
    - start infiniband tear down
    - wait till got RDMA_CM_EVENT_TIMEWAIT_EXIT
    - set socket state to TCP_CLOSE
    - put last socket ref count - this will call sdp_destruct()

2. No need for sdp_time_wait

3. Abortive close will immedietly start infiniband teardown - will finilize the
   socket closing when CM finish.

4. Fixed references accounting + state exchanges

Signed-off-by: Amir Vadai <amirv@mellanox.co.il>
drivers/infiniband/ulp/sdp/sdp.h
drivers/infiniband/ulp/sdp/sdp_bcopy.c
drivers/infiniband/ulp/sdp/sdp_cma.c
drivers/infiniband/ulp/sdp/sdp_main.c

index 5bd404131c84dde89ab8cb9c511e0e21574f1c58..c071e54d0293a941132612411f32766427e3f655 100644 (file)
@@ -110,7 +110,7 @@ struct sdp_sock {
        struct work_struct work;
        wait_queue_head_t wq;
 
-       struct delayed_work time_wait_work;
+       struct delayed_work dreq_wait_work;
        struct work_struct destroy_work;
 
        /* Like tcp_sock */
@@ -125,14 +125,19 @@ struct sdp_sock {
        int xmit_size_goal;
        int nonagle;
 
-       int time_wait;
+       int dreq_wait_timeout;
 
        unsigned keepalive_time;
 
+       spinlock_t lock;
+
        /* tx_head/rx_head when keepalive timer started */
        unsigned keepalive_tx_head;
        unsigned keepalive_rx_head;
 
+       int destructed_already;
+       int sdp_disconnect;
+
        /* Data below will be reset on error */
        /* rdma specific */
        struct rdma_cm_id *id;
@@ -218,24 +223,68 @@ static inline struct sdp_sock *sdp_sk(const struct sock *sk)
                return (struct sdp_sock *)sk;
 }
 
-static inline void sdp_set_state(struct sock *sk, int state)
+static inline char *sdp_state_str(int state)
+{
+       static char *state_str[] = {
+               [TCP_ESTABLISHED] = "TCP_ESTABLISHED",
+               [TCP_SYN_SENT] = "TCP_SYN_SENT",
+               [TCP_SYN_RECV] = "TCP_SYN_RECV",
+               [TCP_FIN_WAIT1] = "TCP_FIN_WAIT1",
+               [TCP_FIN_WAIT2] = "TCP_FIN_WAIT2",
+               [TCP_TIME_WAIT] = "TCP_TIME_WAIT",
+               [TCP_CLOSE] = "TCP_CLOSE",
+               [TCP_CLOSE_WAIT] = "TCP_CLOSE_WAIT",
+               [TCP_LAST_ACK] = "TCP_LAST_ACK",
+               [TCP_LISTEN] = "TCP_LISTEN",
+               [TCP_CLOSING] = "TCP_CLOSING",
+       };
+
+       if (state < 0 || state >= TCP_MAX_STATES)
+               return "unknown";
+
+       return state_str[state];
+}
+
+static inline int _sdp_exch_state(const char *func, int line, struct sock *sk,
+                                int from_states, int state)
 {
+       unsigned long flags;
+       int old;
+
+       spin_lock_irqsave(&sdp_sk(sk)->lock, flags);
+       
+       sdp_dbg(sk, "%s:%d - set state: %s -> %s 0x%x\n", __func__, __LINE__,
+               sdp_state_str(sk->sk_state), sdp_state_str(state), from_states);
+
+       if ((1 << sk->sk_state) & ~from_states) {
+               sdp_warn(sk, "trying to exchange state from unexpected state "
+                       "%s to state %s. expected states: 0x%x\n",
+                       sdp_state_str(sk->sk_state), sdp_state_str(state),
+                       from_states);
+       }
+
+       old = sk->sk_state;
        sk->sk_state = state;
+
+       spin_unlock_irqrestore(&sdp_sk(sk)->lock, flags);
+
+       return old;
 }
+#define sdp_exch_state(sk, from_states, state) \
+       _sdp_exch_state(__func__, __LINE__, sk, from_states, state)
 
 static inline void sdp_set_error(struct sock *sk, int err)
 {
+       int ib_teardown_states = TCPF_FIN_WAIT1 | TCPF_CLOSE_WAIT
+               | TCPF_LAST_ACK;
        sk->sk_err = -err;
        if (sk->sk_socket)
-               sk->sk_socket->state = SS_UNCONNECTED;
+               sk->sk_socket->state = SS_DISCONNECTING;
 
-       sdp_set_state(sk, TCP_CLOSE);
-
-       if (sdp_sk(sk)->time_wait) {
-               sdp_dbg(sk, "%s: destroy in time wait state\n", __func__);
-               sdp_sk(sk)->time_wait = 0;
-               queue_work(sdp_workqueue, &sdp_sk(sk)->destroy_work);
-       }
+       if ((1 << sk->sk_state) & ib_teardown_states)
+               sdp_exch_state(sk, ib_teardown_states, TCP_TIME_WAIT);
+       else
+               sdp_exch_state(sk, ~0, TCP_CLOSE);
 
        sk->sk_error_report(sk);
 }
@@ -245,7 +294,6 @@ extern struct workqueue_struct *sdp_workqueue;
 int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *);
 void sdp_reset(struct sock *sk);
 void sdp_reset_sk(struct sock *sk, int rc);
-void sdp_time_wait_destroy_sk(struct sdp_sock *ssk);
 void sdp_completion_handler(struct ib_cq *cq, void *cq_context);
 void sdp_work(struct work_struct *work);
 int sdp_post_credits(struct sdp_sock *ssk);
@@ -254,7 +302,8 @@ void sdp_post_recvs(struct sdp_sock *ssk);
 int sdp_poll_cq(struct sdp_sock *ssk, struct ib_cq *cq);
 void sdp_post_sends(struct sdp_sock *ssk, int nonagle);
 void sdp_destroy_work(struct work_struct *work);
-void sdp_time_wait_work(struct work_struct *work);
+void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk);
+void sdp_dreq_wait_timeout_work(struct work_struct *work);
 struct sk_buff *sdp_recv_completion(struct sdp_sock *ssk, int id);
 struct sk_buff *sdp_send_completion(struct sdp_sock *ssk, int mseq);
 void sdp_urg(struct sdp_sock *ssk, struct sk_buff *skb);
index 3a8d5acfb506a6eb483be329147c720980305f86..7553003a9bcc809ff29bcdb6510908a4fde5bfc9 100644 (file)
@@ -96,7 +96,7 @@ void sdp_remove_large_sock(struct sdp_sock *ssk)
        }
 }
 
-/* Like tcp_fin */
+/* Like tcp_fin - called when SDP_MID_DISCONNECT is received */
 static void sdp_fin(struct sock *sk)
 {
        sdp_dbg(sk, "%s\n", __func__);
@@ -104,6 +104,36 @@ static void sdp_fin(struct sock *sk)
        sk->sk_shutdown |= RCV_SHUTDOWN;
        sock_set_flag(sk, SOCK_DONE);
 
+       switch (sk->sk_state) {
+       case TCP_SYN_RECV:
+       case TCP_ESTABLISHED:
+               sdp_exch_state(sk, TCPF_SYN_RECV | TCPF_ESTABLISHED,
+                               TCP_CLOSE_WAIT);
+               break;
+
+       case TCP_FIN_WAIT1:
+               /* Received a reply FIN - start Infiniband tear down */
+               sdp_dbg(sk, "%s: Starting Infiniband tear down sending DREQ\n",
+                               __func__);
+               sdp_exch_state(sk, TCPF_FIN_WAIT1, TCP_TIME_WAIT);
+
+               if (sdp_sk(sk)->id) {
+                       rdma_disconnect(sdp_sk(sk)->id);
+               } else {
+                       sdp_warn(sk, "%s: sdp_sk(sk)->id is NULL\n", __func__);
+                       BUG();
+               }
+               break;
+       case TCP_TIME_WAIT:
+               /* This is a mutual close situation and we've got the DREQ from
+                  the peer before the SDP_MID_DISCONNECT */
+               break;
+       default:
+               sdp_warn(sk, "%s: FIN in unexpected state. sk->sk_state=%d\n",
+                               __func__, sk->sk_state);
+               break;
+       }
+
 
        sk_mem_reclaim(sk);
 
@@ -389,7 +419,7 @@ static inline struct sk_buff *sdp_sock_queue_rcv_skb(struct sock *sk,
 {
        int skb_len;
        struct sdp_sock *ssk = sdp_sk(sk);
-       struct sk_buff *tail;
+       struct sk_buff *tail = NULL;
 
        /* not needed since sk_rmem_alloc is not currently used
         * TODO - remove this?
@@ -524,7 +554,9 @@ void sdp_post_sends(struct sdp_sock *ssk, int nonagle)
 
        if (unlikely(c < ssk->rx_head - ssk->rx_tail) &&
            likely(ssk->bufs > 1) &&
-           likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) {
+           likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) &&
+           likely((1 << ssk->isk.sk.sk_state) &
+                   (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) {
                skb = sdp_stream_alloc_skb(&ssk->isk.sk,
                                          sizeof(struct sdp_bsdh),
                                          GFP_KERNEL);
@@ -533,20 +565,16 @@ void sdp_post_sends(struct sdp_sock *ssk, int nonagle)
                sdp_post_send(ssk, skb, SDP_MID_DATA);
        }
 
-       if (unlikely((1 << ssk->isk.sk.sk_state) &
-                       (TCPF_FIN_WAIT1 | TCPF_LAST_ACK)) &&
+       if (unlikely(ssk->sdp_disconnect) &&
                !ssk->isk.sk.sk_send_head &&
                ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) {
+               ssk->sdp_disconnect = 0;
                skb = sdp_stream_alloc_skb(&ssk->isk.sk,
                                          sizeof(struct sdp_bsdh),
                                          gfp_page);
                /* FIXME */
                BUG_ON(!skb);
                sdp_post_send(ssk, skb, SDP_MID_DISCONN);
-               if (ssk->isk.sk.sk_state == TCP_FIN_WAIT1)
-                       sdp_set_state(&ssk->isk.sk, TCP_FIN_WAIT2);
-               else
-                       sdp_set_state(&ssk->isk.sk, TCP_CLOSING);
        }
 }
 
@@ -590,6 +618,7 @@ static void sdp_handle_resize_ack(struct sdp_sock *ssk, struct sdp_chrecvbuf *bu
 
 static int sdp_handle_recv_comp(struct sdp_sock *ssk, struct ib_wc *wc)
 {
+       struct sock *sk = &ssk->isk.sk;
        int frags;
        struct sk_buff *skb;
        struct sdp_bsdh *h;
@@ -603,16 +632,15 @@ static int sdp_handle_recv_comp(struct sdp_sock *ssk, struct ib_wc *wc)
 
        if (unlikely(wc->status)) {
                if (wc->status != IB_WC_WR_FLUSH_ERR) {
-                       sdp_dbg(&ssk->isk.sk,
-                                       "Recv completion with error. "
-                                       "Status %d\n", wc->status);
-                       sdp_reset(&ssk->isk.sk);
+                       sdp_dbg(sk, "Recv completion with error. Status %d\n",
+                               wc->status);
+                       sdp_reset(sk);
                }
                __kfree_skb(skb);
                return 0;
        }
 
-       sdp_dbg_data(&ssk->isk.sk, "Recv completion. ID %d Length %d\n",
+       sdp_dbg_data(sk, "Recv completion. ID %d Length %d\n",
                        (int)wc->wr_id, wc->byte_len);
        if (unlikely(wc->byte_len < sizeof(struct sdp_bsdh))) {
                printk(KERN_WARNING "SDP BUG! byte_len %d < %zd\n",
@@ -651,7 +679,7 @@ static int sdp_handle_recv_comp(struct sdp_sock *ssk, struct ib_wc *wc)
        }
 
        if (unlikely(h->flags & SDP_OOB_PEND))
-               sk_send_sigurg(&ssk->isk.sk);
+               sk_send_sigurg(sk);
 
        skb_pull(skb, sizeof(struct sdp_bsdh));
 
@@ -661,21 +689,33 @@ static int sdp_handle_recv_comp(struct sdp_sock *ssk, struct ib_wc *wc)
                        __kfree_skb(skb);
                        break;
                }
-               skb = sdp_sock_queue_rcv_skb(&ssk->isk.sk, skb);
+
+               if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
+                       /* got data in RCV_SHUTDOWN */
+                       if (sk->sk_state == TCP_FIN_WAIT1) {
+                               /* go into abortive close */
+                               sdp_exch_state(sk, TCPF_FIN_WAIT1,
+                                              TCP_TIME_WAIT);
+
+                               sk->sk_prot->disconnect(sk, 0);
+                       }
+
+                       __kfree_skb(skb);
+                       break;
+               }
+               skb = sdp_sock_queue_rcv_skb(sk, skb);
                if (unlikely(h->flags & SDP_OOB_PRES))
                        sdp_urg(ssk, skb);
                break;
        case SDP_MID_DISCONN:
-               /* this will wake recvmsg */
-               sdp_sock_queue_rcv_skb(&ssk->isk.sk, skb);
-               sdp_fin(&ssk->isk.sk);
+               __kfree_skb(skb);
+               sdp_fin(sk);
                break;
        case SDP_MID_CHRCVBUF:
                sdp_handle_resize_request(ssk,
-                               (struct sdp_chrecvbuf *)skb->data);
+                       (struct sdp_chrecvbuf *)skb->data);
                __kfree_skb(skb);
                break;
-
        case SDP_MID_CHRCVBUF_ACK:
                sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)skb->data);
                __kfree_skb(skb);
@@ -685,27 +725,46 @@ static int sdp_handle_recv_comp(struct sdp_sock *ssk, struct ib_wc *wc)
                printk(KERN_WARNING "SDP: FIXME MID %d\n", h->mid);
                __kfree_skb(skb);
        }
+
        return 0;
 }
 
 static int sdp_handle_send_comp(struct sdp_sock *ssk, struct ib_wc *wc)
 {
        struct sk_buff *skb;
+       struct sdp_bsdh *h;
 
        skb = sdp_send_completion(ssk, wc->wr_id);
        if (unlikely(!skb))
                return -1;
-       sk_wmem_free_skb(&ssk->isk.sk, skb);
+
        if (unlikely(wc->status)) {
                if (wc->status != IB_WC_WR_FLUSH_ERR) {
-                       sdp_dbg(&ssk->isk.sk,
-                                       "Send completion with error. "
-                                       "Status %d\n", wc->status);
-                       sdp_set_error(&ssk->isk.sk, -ECONNRESET);
+                       struct sock *sk = &ssk->isk.sk;
+                       sdp_dbg(sk, "Send completion with error. "
+                               "Status %d\n", wc->status);
+                       sdp_set_error(sk, -ECONNRESET);
                        wake_up(&ssk->wq);
+
+                       queue_work(sdp_workqueue, &ssk->destroy_work);
                }
+               goto out;
+       }
+
+       h = (struct sdp_bsdh *)skb->data;
+
+       if (likely(h->mid != SDP_MID_DISCONN))
+               goto out;
+
+       if ((1 << ssk->isk.sk.sk_state) & ~(TCPF_FIN_WAIT1 | TCPF_LAST_ACK)) {
+               sdp_dbg(&ssk->isk.sk,
+                       "%s: sent DISCONNECT from unexpected state %d\n",
+                       __func__, ssk->isk.sk.sk_state);
        }
 
+out:
+       sk_wmem_free_skb(&ssk->isk.sk, skb);
+
        return 0;
 }
 
@@ -734,13 +793,6 @@ static void sdp_handle_wc(struct sdp_sock *ssk, struct ib_wc *wc)
 
                return;
        }
-
-       if (ssk->time_wait && !ssk->isk.sk.sk_send_head &&
-           ssk->tx_head == ssk->tx_tail) {
-               sdp_dbg(&ssk->isk.sk, "%s: destroy in time wait state\n",
-                       __func__);
-               sdp_time_wait_destroy_sk(ssk);
-       }
 }
 
 void sdp_completion_handler(struct ib_cq *cq, void *cq_context)
index f2fb0831ace990b2eebbe31fb233628e23252e4f..b84fb750689d1a9becb63b97526d3879e227fbb5 100644 (file)
@@ -220,7 +220,7 @@ int sdp_connect_handler(struct sock *sk, struct rdma_cm_id *id,
        sdp_add_sock(sdp_sk(child));
        INIT_LIST_HEAD(&sdp_sk(child)->accept_queue);
        INIT_LIST_HEAD(&sdp_sk(child)->backlog_queue);
-       INIT_DELAYED_WORK(&sdp_sk(child)->time_wait_work, sdp_time_wait_work);
+       INIT_DELAYED_WORK(&sdp_sk(child)->dreq_wait_work, sdp_dreq_wait_timeout_work);
        INIT_WORK(&sdp_sk(child)->destroy_work, sdp_destroy_work);
 
        dst_addr = (struct sockaddr_in *)&id->route.addr.dst_addr;
@@ -256,7 +256,7 @@ int sdp_connect_handler(struct sock *sk, struct rdma_cm_id *id,
        list_add_tail(&sdp_sk(child)->backlog_queue, &sdp_sk(sk)->backlog_queue);
        sdp_sk(child)->parent = sk;
 
-       sdp_set_state(child, TCP_SYN_RECV);
+       sdp_exch_state(child, TCPF_LISTEN | TCPF_CLOSE, TCP_SYN_RECV);
 
        /* child->sk_write_space(child); */
        /* child->sk_data_ready(child, 0); */
@@ -272,7 +272,7 @@ static int sdp_response_handler(struct sock *sk, struct rdma_cm_id *id,
        struct sockaddr_in *dst_addr;
        sdp_dbg(sk, "%s\n", __func__);
 
-       sdp_set_state(sk, TCP_ESTABLISHED);
+       sdp_exch_state(sk, TCPF_SYN_SENT, TCP_ESTABLISHED);
 
        if (sock_flag(sk, SOCK_KEEPOPEN))
                sdp_start_keepalive_timer(sk);
@@ -316,7 +316,7 @@ int sdp_connected_handler(struct sock *sk, struct rdma_cm_event *event)
        parent = sdp_sk(sk)->parent;
        BUG_ON(!parent);
 
-       sdp_set_state(sk, TCP_ESTABLISHED);
+       sdp_exch_state(sk, TCPF_SYN_RECV, TCP_ESTABLISHED);
 
        if (sock_flag(sk, SOCK_KEEPOPEN))
                sdp_start_keepalive_timer(sk);
@@ -498,9 +498,28 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
                        ((struct sockaddr_in *)&id->route.addr.src_addr)->sin_addr.s_addr;
                rc = sdp_connected_handler(sk, event);
                break;
-       case RDMA_CM_EVENT_DISCONNECTED:
+       case RDMA_CM_EVENT_DISCONNECTED: /* This means DREQ/DREP received */
                sdp_dbg(sk, "RDMA_CM_EVENT_DISCONNECTED\n");
+
+               if (sk->sk_state == TCP_LAST_ACK) {
+                       if (sdp_sk(sk)->dreq_wait_timeout)
+                               sdp_cancel_dreq_wait_timeout(sdp_sk(sk));
+
+                       sdp_exch_state(sk, TCPF_LAST_ACK, TCP_TIME_WAIT);
+
+                       sdp_dbg(sk, "%s: waiting for Infiniband tear down\n",
+                               __func__);
+               }
+
                rdma_disconnect(id);
+
+               if (sk->sk_state != TCP_TIME_WAIT) {
+                       sdp_set_error(sk, EPIPE);
+                       rc = sdp_disconnected_handler(sk);
+               }
+               break;
+       case RDMA_CM_EVENT_TIMWAIT_EXIT:
+               sdp_dbg(sk, "RDMA_CM_EVENT_TIMEWAIT_EXIT\n");
                rc = sdp_disconnected_handler(sk);
                break;
        case RDMA_CM_EVENT_DEVICE_REMOVAL:
@@ -517,6 +536,12 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
        sdp_dbg(sk, "%s event %d handled\n", __func__, event->event);
 
        if (rc && sdp_sk(sk)->id == id) {
+               if (sk->sk_state == TCP_SYN_RECV) {
+                       /* sdp_close() will not be called therefore we need
+                          to take a refernce till infiniband teardown is
+                          finished */
+                       sock_hold(sk);
+               }
                child = sk;
                sdp_sk(sk)->id = NULL;
                id->qp = NULL;
index acd3aabdc4f3d2df0a31a1ca8bc260cda70985fb..c3bd9f4c686d57d326f95b5d6e97cbcd3026f815 100644 (file)
@@ -342,13 +342,10 @@ void sdp_reset_sk(struct sock *sk, int rc)
 
        memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), id));
 
-       if (ssk->time_wait) {
-               sdp_dbg(sk, "%s: destroy in time wait state\n", __func__);
-               sdp_time_wait_destroy_sk(ssk);
-       }
-
        sk->sk_state_change(sk);
 
+       queue_work(sdp_workqueue, &ssk->destroy_work);
+
        read_unlock(&device_removal_lock);
 }
 
@@ -417,6 +414,12 @@ static void sdp_destruct(struct sock *sk)
        struct sdp_sock *s, *t;
 
        sdp_dbg(sk, "%s\n", __func__);
+       if (ssk->destructed_already) {
+               sdp_warn(sk, "redestructing sk!");
+               return;
+       }
+
+       ssk->destructed_already = 1;
 
        sdp_remove_sock(ssk);
        
@@ -436,14 +439,15 @@ done:
        sdp_dbg(sk, "%s done\n", __func__);
 }
 
-static void sdp_send_active_reset(struct sock *sk, gfp_t priority)
+static void sdp_send_disconnect(struct sock *sk)
 {
-       sk->sk_prot->disconnect(sk, 0);
+       sdp_sk(sk)->sdp_disconnect = 1;
+       sdp_post_sends(sdp_sk(sk), 0);
 }
 
 /*
  *     State processing on a close.
- *     TCP_ESTABLISHED -> TCP_FIN_WAIT1 -> TCP_FIN_WAIT2 -> TCP_CLOSE
+ *     TCP_ESTABLISHED -> TCP_FIN_WAIT1 -> TCP_CLOSE
  */
 
 static int sdp_close_state(struct sock *sk)
@@ -452,10 +456,14 @@ static int sdp_close_state(struct sock *sk)
                return 0;
 
        if (sk->sk_state == TCP_ESTABLISHED)
-               sdp_set_state(sk, TCP_FIN_WAIT1);
-       else if (sk->sk_state == TCP_CLOSE_WAIT)
-               sdp_set_state(sk, TCP_LAST_ACK);
-       else
+               sdp_exch_state(sk, TCPF_ESTABLISHED, TCP_FIN_WAIT1);
+       else if (sk->sk_state == TCP_CLOSE_WAIT) {
+               sdp_exch_state(sk, TCPF_CLOSE_WAIT, TCP_LAST_ACK);
+
+               sdp_sk(sk)->dreq_wait_timeout = 1;
+               queue_delayed_work(sdp_workqueue, &sdp_sk(sk)->dreq_wait_work,
+                                  TCP_FIN_TIMEOUT);
+       } else
                return 0;
        return 1;
 }
@@ -473,14 +481,23 @@ static void sdp_close(struct sock *sk, long timeout)
        sdp_delete_keepalive_timer(sk);
 
        sk->sk_shutdown = SHUTDOWN_MASK;
+
+       if ((1 << sk->sk_state) & (TCPF_TIME_WAIT | TCPF_CLOSE)) {
+               /* this could happen if socket was closed by a CM teardown
+                  and after that the user called close() */
+               goto out;
+       }
+
        if (sk->sk_state == TCP_LISTEN || sk->sk_state == TCP_SYN_SENT) {
-               sdp_set_state(sk, TCP_CLOSE);
+               sdp_exch_state(sk, TCPF_LISTEN | TCPF_SYN_SENT, TCP_CLOSE);
 
                /* Special case: stop listening.
                   This is done by sdp_destruct. */
                goto adjudge_to_death;
        }
 
+       sock_hold(sk);
+
        /*  We need to flush the recv. buffs.  We do this only on the
         *  descriptor close, not protocol-sourced closes, because the
         *  reader process may not have drained the data yet!
@@ -501,27 +518,28 @@ static void sdp_close(struct sock *sk, long timeout)
         * the FTP client, wheee...  Note: timeout is always zero
         * in such a case.
         */
-       if (data_was_unread) {
+       if (data_was_unread ||
+               (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime)) {
                /* Unread data was tossed, zap the connection. */
                NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
-               sdp_set_state(sk, TCP_CLOSE);
-               sdp_send_active_reset(sk, GFP_KERNEL);
-       } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
-               /* Check zero linger _after_ checking for unread data. */
+               sdp_exch_state(sk, TCPF_CLOSE_WAIT | TCPF_ESTABLISHED,
+                              TCP_TIME_WAIT);
+
+               /* Go into abortive close */
                sk->sk_prot->disconnect(sk, 0);
-               NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
        } else if (sdp_close_state(sk)) {
                /* We FIN if the application ate all the data before
                 * zapping the connection.
                 */
 
-               sdp_post_sends(sdp_sk(sk), 0);
+               sdp_send_disconnect(sk);
        }
 
        /* TODO: state should move to CLOSE or CLOSE_WAIT etc on disconnect.
           Since it currently doesn't, do it here to avoid blocking below. */
        if (!sdp_sk(sk)->id)
-               sdp_set_state(sk, TCP_CLOSE);
+               sdp_exch_state(sk, TCPF_FIN_WAIT1 | TCPF_LAST_ACK |
+                              TCPF_CLOSE_WAIT, TCP_CLOSE);
 
        sk_stream_wait_close(sk, timeout);
 
@@ -533,7 +551,6 @@ adjudge_to_death:
         */
        lock_sock(sk);
 
-       sock_hold(sk);
        sock_orphan(sk);
 
        /*      This is a (useful) BSD violating of the RFC. There is a
@@ -549,35 +566,21 @@ adjudge_to_death:
         *      consume significant resources. Let's do it with special
         *      linger2 option.                                 --ANK
         */
-
-       if (sk->sk_state == TCP_FIN_WAIT2 &&
-               !sk->sk_send_head &&
-               sdp_sk(sk)->tx_head == sdp_sk(sk)->tx_tail) {
-               sdp_set_state(sk, TCP_CLOSE);
-       }
-
-       if ((1 << sk->sk_state) & (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)) {
-               sdp_sk(sk)->time_wait = 1;
+       if (sk->sk_state == TCP_FIN_WAIT1) {
                /* TODO: liger2 unimplemented.
                   We should wait 3.5 * rto. How do I know rto? */
                /* TODO: tcp_fin_time to get timeout */
                sdp_dbg(sk, "%s: entering time wait refcnt %d\n", __func__,
                        atomic_read(&sk->sk_refcnt));
                atomic_inc(sk->sk_prot->orphan_count);
-               queue_delayed_work(sdp_workqueue, &sdp_sk(sk)->time_wait_work,
-                                  TCP_FIN_TIMEOUT);
-               goto out;
        }
 
        /* TODO: limit number of orphaned sockets.
           TCP has sysctl_tcp_mem and sysctl_tcp_max_orphans */
-       sock_put(sk);
 
-       /* Otherwise, socket is reprieved until protocol close. */
 out:
-       sdp_dbg(sk, "%s: last socket put %d\n", __func__,
-               atomic_read(&sk->sk_refcnt));
        release_sock(sk);
+
        sk_common_release(sk);
 }
 
@@ -622,7 +625,7 @@ static int sdp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
                return rc;
        }
 
-       sdp_set_state(sk, TCP_SYN_SENT);
+       sdp_exch_state(sk, TCPF_CLOSE, TCP_SYN_SENT);
        return 0;
 }
 
@@ -635,13 +638,15 @@ static int sdp_disconnect(struct sock *sk, int flags)
        struct rdma_cm_id *id;
 
        sdp_dbg(sk, "%s\n", __func__);
-       if (ssk->id)
-               rc = rdma_disconnect(ssk->id);
 
-       if (old_state != TCP_LISTEN)
+       if (old_state != TCP_LISTEN) {
+               if (ssk->id)
+                       rc = rdma_disconnect(ssk->id);
+
                return rc;
+       }
 
-       sdp_set_state(sk, TCP_CLOSE);
+       sdp_exch_state(sk, TCPF_LISTEN, TCP_CLOSE);
        id = ssk->id;
        ssk->id = NULL;
        release_sock(sk); /* release socket since locking semantics is parent
@@ -827,45 +832,53 @@ static int sdp_ioctl(struct sock *sk, int cmd, unsigned long arg)
        return put_user(answ, (int __user *)arg); 
 }
 
+void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk)
+{
+       ssk->dreq_wait_timeout = 0;
+       cancel_delayed_work(&ssk->dreq_wait_work);
+       atomic_dec(ssk->isk.sk.sk_prot->orphan_count);
+}
+
 void sdp_destroy_work(struct work_struct *work)
 {
        struct sdp_sock *ssk = container_of(work, struct sdp_sock, destroy_work);
        struct sock *sk = &ssk->isk.sk;
        sdp_dbg(sk, "%s: refcnt %d\n", __func__, atomic_read(&sk->sk_refcnt));
 
-       cancel_delayed_work(&sdp_sk(sk)->time_wait_work);
-       atomic_dec(sk->sk_prot->orphan_count);
+       if (ssk->dreq_wait_timeout)
+               sdp_cancel_dreq_wait_timeout(ssk);
 
-       sock_put(sk);
+       if (sk->sk_state == TCP_TIME_WAIT)
+               sock_put(sk);
+
+       /* In normal close current state is TCP_TIME_WAIT or TCP_CLOSE
+          but if a CM connection is dropped below our legs state could
+          be any state */
+       sdp_exch_state(sk, ~0, TCP_CLOSE);
 }
 
-void sdp_time_wait_work(struct work_struct *work)
+void sdp_dreq_wait_timeout_work(struct work_struct *work)
 {
-       struct sdp_sock *ssk = container_of(work, struct sdp_sock, time_wait_work.work);
+       struct sdp_sock *ssk =
+               container_of(work, struct sdp_sock, dreq_wait_work.work);
        struct sock *sk = &ssk->isk.sk;
+
        lock_sock(sk);
-       sdp_dbg(sk, "%s\n", __func__);
 
-       if (!sdp_sk(sk)->time_wait) {
+       if (!sdp_sk(sk)->dreq_wait_timeout) {
                release_sock(sk);
                return;
        }
 
-       sdp_dbg(sk, "%s: refcnt %d\n", __func__, atomic_read(&sk->sk_refcnt));
+       sdp_dbg(sk, "%s: timed out waiting for DREQ\n", __func__);
 
-       sdp_set_state(sk, TCP_CLOSE);
-       sdp_sk(sk)->time_wait = 0;
-       release_sock(sk);
+       sdp_sk(sk)->dreq_wait_timeout = 0;
+       sdp_exch_state(sk, TCPF_LAST_ACK, TCP_TIME_WAIT);
 
-       atomic_dec(sk->sk_prot->orphan_count);
-       sock_put(sk);
-}
+       release_sock(sk);
 
-void sdp_time_wait_destroy_sk(struct sdp_sock *ssk)
-{
-       ssk->time_wait = 0;
-       sdp_set_state(&ssk->isk.sk, TCP_CLOSE);
-       queue_work(sdp_workqueue, &ssk->destroy_work);
+       if (sdp_sk(sk)->id)
+               rdma_disconnect(sdp_sk(sk)->id);
 }
 
 static int sdp_init_sock(struct sock *sk)
@@ -876,10 +889,15 @@ static int sdp_init_sock(struct sock *sk)
 
        INIT_LIST_HEAD(&ssk->accept_queue);
        INIT_LIST_HEAD(&ssk->backlog_queue);
-       INIT_DELAYED_WORK(&ssk->time_wait_work, sdp_time_wait_work);
+       INIT_DELAYED_WORK(&ssk->dreq_wait_work, sdp_dreq_wait_timeout_work);
        INIT_WORK(&ssk->destroy_work, sdp_destroy_work);
 
        sk->sk_route_caps |= NETIF_F_SG | NETIF_F_NO_CSUM;
+
+       ssk->sdp_disconnect = 0;
+       ssk->destructed_already = 0;
+       spin_lock_init(&ssk->lock);
+
        return 0;
 }
 
@@ -891,15 +909,8 @@ static void sdp_shutdown(struct sock *sk, int how)
        if (!(how & SEND_SHUTDOWN))
                return;
 
-       if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
-               return;
-
-       if (sk->sk_state == TCP_ESTABLISHED)
-               sdp_set_state(sk, TCP_FIN_WAIT1);
-       else if (sk->sk_state == TCP_CLOSE_WAIT)
-               sdp_set_state(sk, TCP_LAST_ACK);
-       else
-               return;
+       if (!sdp_close_state(sk))
+           return;
 
        /*
         * Just turn off CORK here.
@@ -910,7 +921,7 @@ static void sdp_shutdown(struct sock *sk, int how)
        if (ssk->nonagle & TCP_NAGLE_OFF)
                ssk->nonagle |= TCP_NAGLE_PUSH;
 
-       sdp_post_sends(ssk, 0);
+       sdp_send_disconnect(sk);
 }
 
 static void sdp_mark_push(struct sdp_sock *ssk, struct sk_buff *skb)
@@ -1991,7 +2002,7 @@ static int sdp_listen(struct sock *sk, int backlog)
                sdp_warn(sk, "rdma_listen failed: %d\n", rc);
                sdp_set_error(sk, rc);
        } else
-               sdp_set_state(sk, TCP_LISTEN);
+               sdp_exch_state(sk, TCPF_CLOSE, TCP_LISTEN);
        return rc;
 }