sdp: Limit FMR resources

- Make FMR pool size a module parameter and set default value to a
  smaller value (20 FMR's)
- Fallback to SendSM silently when  can't allocate FMR

Signed-off-by: Amir Vadai <amirv@mellanox.co.il>

diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h
index 908ee1f9306d3335a222fbac7c8a43082799bdfe..fd98ca905024b929dfd6b82c816bea09c5c00868 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp.h
+++ b/drivers/infiniband/ulp/sdp/sdp.h
@@ -32,8 +32,6 @@
 #define SDP_RX_SIZE 0x40
 
 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64))
-#define SDP_FMR_POOL_SIZE      1024
-#define SDP_FMR_DIRTY_SIZE     ( SDP_FMR_POOL_SIZE / 4 )
 
 #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2))
 
@@ -740,6 +738,7 @@ struct sdpstats {
        u32 zcopy_cross_send;
        u32 zcopy_tx_aborted;
        u32 zcopy_tx_error;
+       u32 fmr_alloc_error;
 };
 
 static inline void sdpstats_hist(u32 *h, u32 val, u32 maxidx, int is_log)

diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c
index ec2428de45f0e27e9f30e518f4174735e3564894..7521478a7f085116365f07ca2ac3bb1882f08631 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp_main.c
+++ b/drivers/infiniband/ulp/sdp/sdp_main.c
@@ -87,6 +87,9 @@ SDP_MODPARAM_INT(sdp_data_debug_level, 0,
                "Enable data path debug tracing if > 0.");
 #endif
 
+SDP_MODPARAM_SINT(sdp_fmr_pool_size, 20, "Number of FMRs to allocate for pool");
+SDP_MODPARAM_SINT(sdp_fmr_dirty_wm, 5, "Watermark to flush fmr pool");
+
 SDP_MODPARAM_SINT(recv_poll, 10, "How many msec to poll recv.");
 SDP_MODPARAM_SINT(sdp_keepalive_time, SDP_KEEPALIVE_TIME,
        "Default idle time in seconds before keepalive probe sent.");
@@ -2744,8 +2747,8 @@ static void sdp_add_device(struct ib_device *device)
         }
 
        memset(&fmr_param, 0, sizeof fmr_param);
-       fmr_param.pool_size         = SDP_FMR_POOL_SIZE;
-       fmr_param.dirty_watermark   = SDP_FMR_DIRTY_SIZE;
+       fmr_param.pool_size         = sdp_fmr_pool_size;
+       fmr_param.dirty_watermark   = sdp_fmr_dirty_wm;
        fmr_param.cache             = 1;
        fmr_param.max_pages_per_fmr = SDP_FMR_SIZE;
        fmr_param.page_shift        = PAGE_SHIFT;

diff --git a/drivers/infiniband/ulp/sdp/sdp_proc.c b/drivers/infiniband/ulp/sdp/sdp_proc.c
index 5e5360fc7febb03159d3e18564eadbd6bea28f99..a074982a055322117c270ef8c620d41b43434a48 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp_proc.c
+++ b/drivers/infiniband/ulp/sdp/sdp_proc.c
@@ -334,6 +334,7 @@ static int sdpstats_seq_show(struct seq_file *seq, void *v)
        seq_printf(seq, "- TX cross send\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_cross_send));
        seq_printf(seq, "- TX aborted by peer\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_aborted));
        seq_printf(seq, "- TX error\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_error));
+       seq_printf(seq, "- FMR alloc error\t: %d\n", SDPSTATS_COUNTER_GET(fmr_alloc_error));
 
        __sdpstats_seq_hist_pcpu(seq, "CPU sendmsg", sendmsg);
        __sdpstats_seq_hist_pcpu(seq, "CPU recvmsg", recvmsg);

diff --git a/drivers/infiniband/ulp/sdp/sdp_zcopy.c b/drivers/infiniband/ulp/sdp/sdp_zcopy.c
index 982eaa347ac92194f8bbd17ef662d9af992a49c2..8ff1e7e848b6d5a84c3371cec7c0b2a391a0e43e 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp_zcopy.c
+++ b/drivers/infiniband/ulp/sdp/sdp_zcopy.c
@@ -481,7 +481,9 @@ static int sdp_alloc_fmr(struct sock *sk, void *uaddr, size_t len,
 
        fmr = ib_fmr_pool_map_phys(sdp_sk(sk)->sdp_dev->fmr_pool, pages, n, 0);
        if (IS_ERR(fmr)) {
-               sdp_warn(sk, "Error allocating fmr: %ld\n", PTR_ERR(fmr));
+               sdp_dbg_data(sk, "Error allocating fmr: %ld\n", PTR_ERR(fmr));
+               SDPSTATS_COUNTER_INC(fmr_alloc_error);
+               rc = PTR_ERR(fmr);
                goto err_fmr_alloc;
        }
 
@@ -505,8 +507,11 @@ err_umem_get:
 
 void sdp_free_fmr(struct sock *sk, struct ib_pool_fmr **_fmr, struct ib_umem **_umem)
 {
-       if (!sdp_sk(sk)->qp_active)
+       if (!sdp_sk(sk)->qp_active) {
+               sdp_warn(sk, "Trying to free fmr after destroying QP! fmr: %p\n",
+                               *_fmr);
                return;
+       }
 
        ib_fmr_pool_unmap(*_fmr);
        *_fmr = NULL;
@@ -571,7 +576,7 @@ int sdp_rdma_to_iovec(struct sock *sk, struct iovec *iov, struct sk_buff *skb,
 
        rc = sdp_alloc_fmr(sk, iov->iov_base, len, &rx_sa->fmr, &rx_sa->umem);
        if (rc) {
-               sdp_warn(sk, "Error allocating fmr: %d\n", rc);
+               sdp_dbg_data(sk, "Error allocating fmr: %d\n", rc);
                goto err_alloc_fmr;
        }
 
@@ -610,7 +615,7 @@ err_post_send:
 
 err_alloc_fmr:
        if (rc && ssk->qp_active) {
-               sdp_warn(sk, "Couldn't do RDMA - post sendsm\n");
+               sdp_dbg_data(sk, "Couldn't do RDMA - post sendsm\n");
                rx_sa->flags |= RX_SA_ABORTED;
        }
 
@@ -650,7 +655,7 @@ static int do_sdp_sendmsg_zcopy(struct sock *sk, struct tx_srcavail_state *tx_sa
        rc = sdp_alloc_fmr(sk, iov->iov_base, iov->iov_len,
                        &tx_sa->fmr, &tx_sa->umem);
        if (rc) {
-               sdp_warn(sk, "Error allocating fmr: %d\n", rc);
+               sdp_dbg_data(sk, "Error allocating fmr: %d\n", rc);
                goto err_alloc_fmr;
        }