From fb060398515c13d592ccd49f3a6263f9062ae728 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Wed, 18 May 2016 10:44:56 -0700 Subject: [PATCH] RDS: IB: rebuild receive caches when needed RDS IB caches code leaks memory & it have been there from the inception of cache code but we didn't noticed them since caches are not teardown in normal operation paths. But now to support features like variable fragment or connection destroy for ACL, caches needs to be destroyed and rebuild if needed. While freeing the caches is just fine, leaking memory while doing that is bug and needs to be addressed. Thanks to Wengang for spotting this stone age leak. Also the cache rebuild needs to be done only when desired so patch optimises that part as well. Tested-by: Michael Nowak Tested-by: Maria Rodriguez Tested-by: Hong Liu Signed-off-by: Santosh Shilimkar --- net/rds/ib.h | 5 +++-- net/rds/ib_cm.c | 7 +++++- net/rds/ib_recv.c | 54 +++++++++++++++++++++++++---------------------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/net/rds/ib.h b/net/rds/ib.h index d3b1a58d5bec..6351423c4827 100644 --- a/net/rds/ib.h +++ b/net/rds/ib.h @@ -216,7 +216,8 @@ struct rds_ib_connection { /* Protocol version specific information */ unsigned int i_flowctl:1; /* enable/disable flow ctl */ u16 i_frag_sz; /* IB fragment size */ - int8_t i_frag_pages; + u16 i_frag_cache_sz; + u8 i_frag_pages; /* Batched completions */ unsigned int i_unsignaled_wrs; @@ -618,7 +619,7 @@ void rds_ib_recv_exit(void); int rds_ib_recv(struct rds_connection *conn); int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic); void rds_ib_recv_free_caches(struct rds_ib_connection *ic); -void rds_ib_recv_purge_frag_cache(struct rds_ib_connection *ic); +void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic); void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp); void rds_ib_inc_free(struct rds_incoming *inc); int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to); diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 42f4fbd1c2bc..39c8b7e5f971 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -176,6 +176,9 @@ static void rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag) } ic->i_frag_pages = ic->i_frag_sz / PAGE_SIZE; + if (!ic->i_frag_pages) + ic->i_frag_pages = 1; + pr_debug("RDS/IB: conn <%pI4, %pI4,%d>, Frags : {%d,%d,%d}, updated {%d -> %d}\n", &conn->c_laddr, &conn->c_faddr, conn->c_tos, ib_init_frag_size / SZ_1K, ic->i_frag_sz / SZ_1K, dp_frag / SZ_1K, @@ -269,8 +272,10 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even */ rds_ib_send_init_ring(ic); - if (!rds_ib_srq_enabled) + if (!rds_ib_srq_enabled) { + rds_ib_recv_rebuild_caches(ic); rds_ib_recv_init_ring(ic); + } /* Post receive buffers - as a side effect, this will update * the posted credit count. */ diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c index 085691e26e75..4a4c33f38e27 100644 --- a/net/rds/ib_recv.c +++ b/net/rds/ib_recv.c @@ -166,6 +166,14 @@ static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache, } } +/* Detach and free frags */ +static void rds_ib_recv_free_frag(struct rds_page_frag *frag) +{ + rdsdebug("RDS/IB: frag %p page %p\n", frag, sg_page(&frag->f_sg)); + list_del_init(&frag->f_item); + __free_pages(sg_page(&frag->f_sg), get_order(frag->f_sg.length)); +} + void rds_ib_recv_free_caches(struct rds_ib_connection *ic) { struct rds_ib_incoming *inc; @@ -191,40 +199,36 @@ void rds_ib_recv_free_caches(struct rds_ib_connection *ic) list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) { list_del(&frag->f_cache_entry); WARN_ON(!list_empty(&frag->f_item)); + rds_ib_recv_free_frag(frag); + atomic_sub(ic->i_frag_pages, &rds_ib_allocation); kmem_cache_free(rds_ib_frag_slab, frag); + atomic_sub(ic->i_frag_sz / 1024, &ic->i_cache_allocs); + rds_ib_stats_add(s_ib_recv_removed_from_cache, ic->i_frag_sz); } } -/* Called from rds_ib_conn_shutdown path on the way - * towards connection destroy or reconnect +/* Called form rds_ib_conn_complete() and takes action only + * if new connection needs different frag size than what is used. */ -void rds_ib_recv_purge_frag_cache(struct rds_ib_connection *ic) +void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic) { - struct rds_ib_cache_head *head; - struct rds_page_frag *frag, *frag_tmp; - LIST_HEAD(list); - int cpu; - - rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); - rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list); - atomic_set(&ic->i_cache_allocs, 0); - - list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) { - list_del(&frag->f_cache_entry); - WARN_ON(!list_empty(&frag->f_item)); - kmem_cache_free(rds_ib_frag_slab, frag); - rds_ib_stats_add(s_ib_recv_added_to_cache, ic->i_frag_sz); - rds_ib_stats_add(s_ib_recv_removed_from_cache, ic->i_frag_sz); + /* init it with the used frag size */ + if (!ic->i_frag_cache_sz) { + ic->i_frag_cache_sz = ic->i_frag_sz; + return; } - for_each_possible_cpu(cpu) { - head = per_cpu_ptr(ic->i_cache_frags.percpu, cpu); - head->first = NULL; - head->count = 0; - } + /* check if existing cache can be re-used */ + if (ic->i_frag_cache_sz == ic->i_frag_sz) + return; + + /* Now re-build the caches */ + rds_ib_recv_free_caches(ic); + rds_ib_recv_alloc_caches(ic); - ic->i_cache_frags.xfer = NULL; - ic->i_cache_frags.ready = NULL; + pr_debug("RDS/IB: Rebuild caches for ic %p i_cm_id %p, frag{%d->%d}\n", + ic, ic->i_cm_id, ic->i_frag_cache_sz, ic->i_frag_sz); + ic->i_frag_cache_sz = ic->i_frag_sz; } /* fwd decl */ -- 2.50.1