RDS/IB: 4KB receive buffers get posted by mistake on 16KB frag connections.

When connections are at 4KB fragments and then it moves to 16KB frags
(for example during uek2 to uek4 upgrade) we see 4KB buffers getting
posted on 16KB connections. This is happening because the 4KB buffers
(buffers from previous connection before the move to 16KB) are getting
added back to the current connection's (16KB) cache.

We will fix this by doing the following.

1) When the recv buffers get freed/released after either the application
   is done reading it or the socket gets closed (process dies, etc.)
   and RDS/IB decides to add that buffer back into the current cache,
   make sure the frag size matches with that of the current connection.

2) When recv completion reports IB_WC_LOC_LEN_ERR, mark the connection state
   as "buffers need to be rebuilt during reconnection". And at the time of
   reconnect rebuild the cache even though the "frag size of the connection"
   has not changed.

Orabug: 25920916

Reviewed-by: Rama Nichanamatlu <rama.nichanamatlu@oracle.com>
Reviewed-by: Shamir Rabinovitch <shamir.rabinovitch@oracle.com>
Signed-off-by: Venkat Venkatsubra <venkat.x.venkatsubra@oracle.com>

diff --git a/net/rds/ib.h b/net/rds/ib.h
index 8fd3e23369adeb5e0f27cb5349f3972b8a4c3a1a..48c7d6a7fca98ac74966fc52f763c77d959e994a 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -49,6 +49,8 @@
 
 #define RDS_WC_MAX 32
 
+#define        RDS_IB_CLEAN_CACHE      1
+
 extern struct rw_semaphore rds_ib_devices_lock;
 extern struct list_head rds_ib_devices;
 
@@ -213,6 +215,7 @@ struct rds_ib_connection {
        u16                     i_frag_sz;      /* IB fragment size */
        u16                     i_frag_cache_sz;
        u8                      i_frag_pages;
+       u8                      i_flags;
 
        /* Batched completions */
        unsigned int            i_unsignaled_wrs;

diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index ab99a0dc42b44cb58b890e7535b15f11c12f5c02..bdfbb8781c048900e0f4b877032545f42169d515 100644 (file)
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -219,8 +219,11 @@ void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic)
        }
 
        /* check if existing cache can be re-used */
-       if (ic->i_frag_cache_sz == ic->i_frag_sz)
-               return;
+       if ((ic->i_frag_cache_sz == ic->i_frag_sz) &&
+               !(ic->i_flags & RDS_IB_CLEAN_CACHE))
+                       return;
+
+       ic->i_flags &= ~RDS_IB_CLEAN_CACHE;
 
        /* Now re-build the caches */
        rds_ib_recv_free_caches(ic);
@@ -260,8 +263,12 @@ void rds_ib_inc_free(struct rds_incoming *inc)
 
        /* Free attached frags */
        list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
-               list_del_init(&frag->f_item);
-               rds_ib_frag_free(ic, frag);
+               if (frag->f_sg.length != ic->i_frag_sz)
+                       rds_ib_recv_free_frag(frag);
+               else {
+                       list_del_init(&frag->f_item);
+                       rds_ib_frag_free(ic, frag);
+               }
        }
        BUG_ON(!list_empty(&ibinc->ii_frags));
 
@@ -1346,6 +1353,8 @@ void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,
                                pr_warn("RDS/IB: recv completion <%pI4,%pI4,%d> had status %u vendor_err 0x%x, disconnecting and reconnecting\n",
                                        &conn->c_laddr, &conn->c_faddr, conn->c_tos,
                                        wc->status, wc->vendor_err);
+                       if (wc->status == IB_WC_LOC_LEN_ERR)
+                               ic->i_flags |= RDS_IB_CLEAN_CACHE;
                        rds_conn_drop(conn, DR_IB_RECV_COMP_ERR);
                        rds_rtd(RDS_RTD_ERR, "status %u => %s\n", wc->status,
                                rds_ib_wc_status_str(wc->status));