RDS: IB: rebuild receive caches when needed

RDS IB caches code leaks memory & it have been there from the
inception of cache code but we didn't noticed them since caches
are not teardown in normal operation paths. But now to support
features like variable fragment or connection destroy for ACL,
caches needs to be destroyed and rebuild if needed.

While freeing the caches is just fine, leaking memory while
doing that is bug and needs to be addressed. Thanks to Wengang
for spotting this stone age leak. Also the cache rebuild needs
to be done only when desired so patch optimises that part as
well.

Tested-by: Michael Nowak <michael.nowak@oracle.com>
Tested-by: Maria Rodriguez <maria.r.rodriguez@oracle.com>
Tested-by: Hong Liu <hong.x.liu@oracle.com>
Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>

diff --git a/net/rds/ib.h b/net/rds/ib.h
index d3b1a58d5becd9af092682945c62eeb2cae9c78f..6351423c4827fe2a01977b6a79dd0657c9378d7e 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -216,7 +216,8 @@ struct rds_ib_connection {
        /* Protocol version specific information */
        unsigned int            i_flowctl:1;    /* enable/disable flow ctl */
        u16                     i_frag_sz;      /* IB fragment size */
-       int8_t                  i_frag_pages;
+       u16                     i_frag_cache_sz;
+       u8                      i_frag_pages;
 
        /* Batched completions */
        unsigned int            i_unsignaled_wrs;
@@ -618,7 +619,7 @@ void rds_ib_recv_exit(void);
 int rds_ib_recv(struct rds_connection *conn);
 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic);
 void rds_ib_recv_free_caches(struct rds_ib_connection *ic);
-void rds_ib_recv_purge_frag_cache(struct rds_ib_connection *ic);
+void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic);
 void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp);
 void rds_ib_inc_free(struct rds_incoming *inc);
 int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 42f4fbd1c2bc967ba97baf09d938fc6a333ffd62..39c8b7e5f971e9c90cb23f57ba8e32c273881f86 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -176,6 +176,9 @@ static void rds_ib_set_frag_size(struct rds_connection *conn, u16 dp_frag)
        }
 
        ic->i_frag_pages =  ic->i_frag_sz / PAGE_SIZE;
+       if (!ic->i_frag_pages)
+               ic->i_frag_pages = 1;
+
        pr_debug("RDS/IB: conn <%pI4, %pI4,%d>, Frags <init,ic,dp>: {%d,%d,%d}, updated {%d -> %d}\n",
                 &conn->c_laddr, &conn->c_faddr, conn->c_tos,
                 ib_init_frag_size / SZ_1K, ic->i_frag_sz / SZ_1K, dp_frag /  SZ_1K,
@@ -269,8 +272,10 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even
         */
        rds_ib_send_init_ring(ic);
 
-       if (!rds_ib_srq_enabled)
+       if (!rds_ib_srq_enabled) {
+               rds_ib_recv_rebuild_caches(ic);
                rds_ib_recv_init_ring(ic);
+       }
 
        /* Post receive buffers - as a side effect, this will update
         * the posted credit count. */

diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index 085691e26e755c121a06208f8511458cfec40522..4a4c33f38e2732cf551f1552508ffdd9d0368858 100644 (file)
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -166,6 +166,14 @@ static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,
        }
 }
 
+/* Detach and free frags */
+static void rds_ib_recv_free_frag(struct rds_page_frag *frag)
+{
+       rdsdebug("RDS/IB: frag %p page %p\n", frag, sg_page(&frag->f_sg));
+       list_del_init(&frag->f_item);
+       __free_pages(sg_page(&frag->f_sg), get_order(frag->f_sg.length));
+}
+
 void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
 {
        struct rds_ib_incoming *inc;
@@ -191,40 +199,36 @@ void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
        list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
                list_del(&frag->f_cache_entry);
                WARN_ON(!list_empty(&frag->f_item));
+               rds_ib_recv_free_frag(frag);
+               atomic_sub(ic->i_frag_pages, &rds_ib_allocation);
                kmem_cache_free(rds_ib_frag_slab, frag);
+               atomic_sub(ic->i_frag_sz / 1024, &ic->i_cache_allocs);
+               rds_ib_stats_add(s_ib_recv_removed_from_cache, ic->i_frag_sz);
        }
 }
 
-/* Called from rds_ib_conn_shutdown path on the way
- * towards connection destroy or reconnect
+/* Called form rds_ib_conn_complete() and takes action only
+ * if new connection needs different frag size than what is used.
  */
-void rds_ib_recv_purge_frag_cache(struct rds_ib_connection *ic)
+void rds_ib_recv_rebuild_caches(struct rds_ib_connection *ic)
 {
-       struct rds_ib_cache_head *head;
-       struct rds_page_frag *frag, *frag_tmp;
-       LIST_HEAD(list);
-       int cpu;
-
-       rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
-       rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);
-       atomic_set(&ic->i_cache_allocs, 0);
-
-       list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
-               list_del(&frag->f_cache_entry);
-               WARN_ON(!list_empty(&frag->f_item));
-               kmem_cache_free(rds_ib_frag_slab, frag);
-               rds_ib_stats_add(s_ib_recv_added_to_cache, ic->i_frag_sz);
-               rds_ib_stats_add(s_ib_recv_removed_from_cache, ic->i_frag_sz);
+       /* init it with the used frag size */
+       if (!ic->i_frag_cache_sz) {
+               ic->i_frag_cache_sz = ic->i_frag_sz;
+               return;
        }
 
-       for_each_possible_cpu(cpu) {
-               head = per_cpu_ptr(ic->i_cache_frags.percpu, cpu);
-               head->first = NULL;
-               head->count = 0;
-       }
+       /* check if existing cache can be re-used */
+       if (ic->i_frag_cache_sz == ic->i_frag_sz)
+               return;
+
+       /* Now re-build the caches */
+       rds_ib_recv_free_caches(ic);
+       rds_ib_recv_alloc_caches(ic);
 
-       ic->i_cache_frags.xfer = NULL;
-       ic->i_cache_frags.ready = NULL;
+       pr_debug("RDS/IB: Rebuild caches for ic %p i_cm_id %p, frag{%d->%d}\n",
+                ic, ic->i_cm_id, ic->i_frag_cache_sz, ic->i_frag_sz);
+       ic->i_frag_cache_sz = ic->i_frag_sz;
 }
 
 /* fwd decl */