RDS: Use a single TCP socket for both send and receive.

Orabug: 22012202

Backport of upstream commit 3b20fc389705 ("RDS: Use a single TCP
socket for both send and receive.");

Commit f711a6ae062c ("net/rds: RDS-TCP: Always create a new rds_sock
for an incoming connection.") modified rds-tcp so that an incoming SYN
would ignore an existing "client" TCP connection which had the local
port set to the transient port.  The motivation for ignoring the existing
"client" connection in f711a6ae was to avoid race conditions and an
endless duel of reconnect attempts triggered by a restart/abort of one
of the nodes in the TCP connection.

However, having separate sockets for active and passive sides
is avoidable, and the simpler model of a single TCP socket for
both send and receives of all RDS connections associated with
that tcp socket makes for easier observability. We avoid the race
conditions from f711a6ae by attempting reconnects in rds_conn_shutdown
if, and only if, the (new) c_outgoing bit is set for RDS_TRANS_TCP.
The c_outgoing bit is initialized in __rds_conn_create().

A side-effect of re-using the client rds_connection for an incoming
SYN is the potential of encountering duelling SYNs, i.e., we
have an outgoing RDS_CONN_CONNECTING socket when we get the incoming
SYN. The logic to arbitrate this criss-crossing SYN exchange in
rds_tcp_accept_one() has been modified to emulate the BGP state
machine: the smaller IP address should back off from the connection
attempt.

Signed-off-by: Sowmini Varadhan <sowmini.varadhan@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Mukesh Kacker <mukesh.kacker@oracle.com>

diff --git a/net/rds/connection.c b/net/rds/connection.c
index 4538d90e2083c203b6dfa1d0a41d89ff201065f2..0635b2450ef255df01f7911d5d069db693152c18 100644 (file)
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -129,10 +129,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
        struct rds_transport *loop_trans;
        unsigned long flags;
        int ret;
-       struct rds_transport *otrans = trans;
 
-       if (!is_outgoing && otrans->t_type == RDS_TRANS_TCP)
-               goto new_conn;
        rcu_read_lock();
        conn = rds_conn_lookup(head, laddr, faddr, trans, tos);
        if (conn
@@ -151,7 +148,6 @@ static struct rds_connection *__rds_conn_create(struct net *net,
        if (conn)
                goto out;
 
-new_conn:
        conn = kmem_cache_alloc(rds_conn_slab, gfp);
        if (!conn) {
                conn = ERR_PTR(-ENOMEM);
@@ -209,6 +205,7 @@ new_conn:
 
        atomic_set(&conn->c_state, RDS_CONN_DOWN);
        conn->c_send_gen = 0;
+       conn->c_outgoing = (is_outgoing ? 1 : 0);
        conn->c_reconnect_jiffies = 0;
        conn->c_reconnect_start = get_seconds();
        conn->c_reconnect_warn = 1;
@@ -255,22 +252,13 @@ new_conn:
                /* Creating normal conn */
                struct rds_connection *found;
 
-               if (!is_outgoing && otrans->t_type == RDS_TRANS_TCP)
-                       found = NULL;
-               else
-                       found = rds_conn_lookup(head, laddr, faddr, trans, tos);
+               found = rds_conn_lookup(head, laddr, faddr, trans, tos);
                if (found) {
                        trans->conn_free(conn->c_transport_data);
                        kmem_cache_free(rds_conn_slab, conn);
                        conn = found;
                } else {
-                       if ((is_outgoing && otrans->t_type == RDS_TRANS_TCP) ||
-                           (otrans->t_type != RDS_TRANS_TCP)) {
-                               /* Only the active side should be added to
-                                * reconnect list for RDS-TCP.
-                                */
-                               hlist_add_head_rcu(&conn->c_hash_node, head);
-                       }
+                       hlist_add_head_rcu(&conn->c_hash_node, head);
                        rds_cong_add_conn(conn);
                        rds_conn_count++;
                }
@@ -369,11 +357,16 @@ void rds_conn_shutdown(struct rds_connection *conn, int restart)
        rcu_read_lock();
        if (!hlist_unhashed(&conn->c_hash_node) && restart) {
                rcu_read_unlock();
-               rds_rtd(RDS_RTD_CM_EXT,
-                       "queueing reconnect request... <%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
-                       NIPQUAD(conn->c_laddr), NIPQUAD(conn->c_faddr),
-                       conn->c_tos);
-               rds_queue_reconnect(conn);
+               if (conn->c_trans->t_type != RDS_TRANS_TCP ||
+                   conn->c_outgoing == 1) {
+                       rds_rtd(RDS_RTD_CM_EXT,
+                               "queueing reconnect request... "
+                               "<%u.%u.%u.%u,%u.%u.%u.%u,%d>\n",
+                               NIPQUAD(conn->c_laddr),
+                               NIPQUAD(conn->c_faddr),
+                               conn->c_tos);
+                       rds_queue_reconnect(conn);
+               }
        } else {
                rcu_read_unlock();
        }

diff --git a/net/rds/rds.h b/net/rds/rds.h
index 78f628143bc643f00f668cca47e8640a3e7df2d8..fcbe790e4f6787be2d7390b42a58437fb412a1a5 100644 (file)
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -99,7 +99,9 @@ struct rds_connection {
        struct hlist_node       c_hash_node;
        __be32                  c_laddr;
        __be32                  c_faddr;
-       unsigned int            c_loopback:1;
+       unsigned int            c_loopback:1,
+                               c_outgoing:1,
+                               c_pad_to_32:30;
        struct rds_connection   *c_passive;
 
        struct rds_cong_map     *c_lcong;

diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c
index 6fb560607fa94015b21c5a57ae050446885b7869..2f2419274443a5556d2092b46796b0468834095e 100644 (file)
--- a/net/rds/tcp_listen.c
+++ b/net/rds/tcp_listen.c
@@ -109,28 +109,24 @@ int rds_tcp_accept_one(struct socket *sock)
                goto out;
        }
        /* An incoming SYN request came in, and TCP just accepted it.
-        * We always create a new conn for listen side of TCP, and do not
-        * add it to the c_hash_list.
         *
         * If the client reboots, this conn will need to be cleaned up.
         * rds_tcp_state_change() will do that cleanup
         */
        rs_tcp = (struct rds_tcp_connection *)conn->c_transport_data;
-       WARN_ON(!rs_tcp || rs_tcp->t_sock);
-
-       /*
-        * see the comment above rds_queue_delayed_reconnect()
-        */
-       if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
-               if (rds_conn_state(conn) == RDS_CONN_UP)
-                       rds_tcp_stats_inc(s_tcp_listen_closed_stale);
-               else
-                       rds_tcp_stats_inc(s_tcp_connect_raced);
-               rds_conn_drop(conn);
+       if (rs_tcp->t_sock &&
+           ntohl(inet->inet_saddr) < ntohl(inet->inet_daddr)) {
+               struct sock *nsk = new_sock->sk;
+
+               nsk->sk_user_data = NULL;
+               nsk->sk_prot->disconnect(nsk, 0);
+               tcp_done(nsk);
+               new_sock = NULL;
                ret = 0;
                goto out;
        }
 
+       rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING);
        rds_tcp_set_callbacks(new_sock, conn);
        rds_connect_complete(conn);
        new_sock = NULL;