From 26989ee6268bd3724e53e0d393698dc7bd3dceb9 Mon Sep 17 00:00:00 2001 From: Venkat Venkatsubra Date: Mon, 8 May 2017 04:23:13 -0700 Subject: [PATCH] RDS/IB: 4KB receive buffers get posted by mistake on 16KB frag connections. When connections are at 4KB fragments and then it moves to 16KB frags (for example during uek2 to uek4 upgrade) we see 4KB buffers getting posted on 16KB connections. This is happening because the 4KB buffers (buffers from previous connection before the move to 16KB) are getting added back to the current connection's (16KB) cache. We will fix this by doing the following. 1) When the recv buffers get freed/released after either the application is done reading it or the socket gets closed (process dies, etc.) and RDS/IB decides to add that buffer back into the current cache, make sure the frag size matches with that of the current connection. 2) When recv completion reports IB_WC_LOC_LEN_ERR, mark the connection state as "buffers need to be rebuilt during reconnection". And at the time of reconnect rebuild the cache even though the "frag size of the connection" has not changed. Orabug: 25920916 Reviewed-by: Rama Nichanamatlu Reviewed-by: Shamir Rabinovitch Signed-off-by: Venkat Venkatsubra --- net/rds/ib.h | 3 +++ net/rds/ib_recv.c | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/net/rds/ib.h b/net/rds/ib.h index 8fd3e23369ade..48c7d6a7fca98 100644 --- a/net/rds/ib.h +++ b/net/rds/ib.h @@ -49,6 +49,8 @@ #define RDS_WC_MAX 32 +#define RDS_IB_CLEAN_CACHE 1 + extern struct rw_semaphore rds_ib_devices_lock; extern struct list_head rds_ib_devices; @@ -213,6 +215,7 @@ struct rds_ib_connection { u16 i_frag_sz; /* IB fragment size */ u16 i_frag_cache_sz; u8 i_frag_pages; + u8 i_flags; /* Batched completions */ unsigned int i_unsignaled_wrs; diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c index ab99a0dc42b44..bdfbb8781c048 100644 --- a/net/rds/ib_recv.c +++ b/net/rds/ib_recv.c @@ -219,8 +219,11 @@ void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic) } /* check if existing cache can be re-used */ - if (ic->i_frag_cache_sz == ic->i_frag_sz) - return; + if ((ic->i_frag_cache_sz == ic->i_frag_sz) && + !(ic->i_flags & RDS_IB_CLEAN_CACHE)) + return; + + ic->i_flags &= ~RDS_IB_CLEAN_CACHE; /* Now re-build the caches */ rds_ib_recv_free_caches(ic); @@ -260,8 +263,12 @@ void rds_ib_inc_free(struct rds_incoming *inc) /* Free attached frags */ list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) { - list_del_init(&frag->f_item); - rds_ib_frag_free(ic, frag); + if (frag->f_sg.length != ic->i_frag_sz) + rds_ib_recv_free_frag(frag); + else { + list_del_init(&frag->f_item); + rds_ib_frag_free(ic, frag); + } } BUG_ON(!list_empty(&ibinc->ii_frags)); @@ -1346,6 +1353,8 @@ void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, pr_warn("RDS/IB: recv completion <%pI4,%pI4,%d> had status %u vendor_err 0x%x, disconnecting and reconnecting\n", &conn->c_laddr, &conn->c_faddr, conn->c_tos, wc->status, wc->vendor_err); + if (wc->status == IB_WC_LOC_LEN_ERR) + ic->i_flags |= RDS_IB_CLEAN_CACHE; rds_conn_drop(conn, DR_IB_RECV_COMP_ERR); rds_rtd(RDS_RTD_ERR, "status %u => %s\n", wc->status, rds_ib_wc_status_str(wc->status)); -- 2.50.1