From: Amir Vadai Date: Tue, 22 Jun 2010 07:45:58 +0000 (+0300) Subject: sdp: Limit FMR resources X-Git-Tag: v4.1.12-92~264^2~5^2~159 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=31885b55221b27a87166c1447a04aeb64c63630d;p=users%2Fjedix%2Flinux-maple.git sdp: Limit FMR resources - Make FMR pool size a module parameter and set default value to a smaller value (20 FMR's) - Fallback to SendSM silently when can't allocate FMR Signed-off-by: Amir Vadai --- diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h index 908ee1f9306d3..fd98ca905024b 100644 --- a/drivers/infiniband/ulp/sdp/sdp.h +++ b/drivers/infiniband/ulp/sdp/sdp.h @@ -32,8 +32,6 @@ #define SDP_RX_SIZE 0x40 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64)) -#define SDP_FMR_POOL_SIZE 1024 -#define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 ) #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2)) @@ -740,6 +738,7 @@ struct sdpstats { u32 zcopy_cross_send; u32 zcopy_tx_aborted; u32 zcopy_tx_error; + u32 fmr_alloc_error; }; static inline void sdpstats_hist(u32 *h, u32 val, u32 maxidx, int is_log) diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c index ec2428de45f0e..7521478a7f085 100644 --- a/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/drivers/infiniband/ulp/sdp/sdp_main.c @@ -87,6 +87,9 @@ SDP_MODPARAM_INT(sdp_data_debug_level, 0, "Enable data path debug tracing if > 0."); #endif +SDP_MODPARAM_SINT(sdp_fmr_pool_size, 20, "Number of FMRs to allocate for pool"); +SDP_MODPARAM_SINT(sdp_fmr_dirty_wm, 5, "Watermark to flush fmr pool"); + SDP_MODPARAM_SINT(recv_poll, 10, "How many msec to poll recv."); SDP_MODPARAM_SINT(sdp_keepalive_time, SDP_KEEPALIVE_TIME, "Default idle time in seconds before keepalive probe sent."); @@ -2744,8 +2747,8 @@ static void sdp_add_device(struct ib_device *device) } memset(&fmr_param, 0, sizeof fmr_param); - fmr_param.pool_size = SDP_FMR_POOL_SIZE; - fmr_param.dirty_watermark = SDP_FMR_DIRTY_SIZE; + fmr_param.pool_size = sdp_fmr_pool_size; + fmr_param.dirty_watermark = sdp_fmr_dirty_wm; fmr_param.cache = 1; fmr_param.max_pages_per_fmr = SDP_FMR_SIZE; fmr_param.page_shift = PAGE_SHIFT; diff --git a/drivers/infiniband/ulp/sdp/sdp_proc.c b/drivers/infiniband/ulp/sdp/sdp_proc.c index 5e5360fc7febb..a074982a05532 100644 --- a/drivers/infiniband/ulp/sdp/sdp_proc.c +++ b/drivers/infiniband/ulp/sdp/sdp_proc.c @@ -334,6 +334,7 @@ static int sdpstats_seq_show(struct seq_file *seq, void *v) seq_printf(seq, "- TX cross send\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_cross_send)); seq_printf(seq, "- TX aborted by peer\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_aborted)); seq_printf(seq, "- TX error\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_error)); + seq_printf(seq, "- FMR alloc error\t: %d\n", SDPSTATS_COUNTER_GET(fmr_alloc_error)); __sdpstats_seq_hist_pcpu(seq, "CPU sendmsg", sendmsg); __sdpstats_seq_hist_pcpu(seq, "CPU recvmsg", recvmsg); diff --git a/drivers/infiniband/ulp/sdp/sdp_zcopy.c b/drivers/infiniband/ulp/sdp/sdp_zcopy.c index 982eaa347ac92..8ff1e7e848b6d 100644 --- a/drivers/infiniband/ulp/sdp/sdp_zcopy.c +++ b/drivers/infiniband/ulp/sdp/sdp_zcopy.c @@ -481,7 +481,9 @@ static int sdp_alloc_fmr(struct sock *sk, void *uaddr, size_t len, fmr = ib_fmr_pool_map_phys(sdp_sk(sk)->sdp_dev->fmr_pool, pages, n, 0); if (IS_ERR(fmr)) { - sdp_warn(sk, "Error allocating fmr: %ld\n", PTR_ERR(fmr)); + sdp_dbg_data(sk, "Error allocating fmr: %ld\n", PTR_ERR(fmr)); + SDPSTATS_COUNTER_INC(fmr_alloc_error); + rc = PTR_ERR(fmr); goto err_fmr_alloc; } @@ -505,8 +507,11 @@ err_umem_get: void sdp_free_fmr(struct sock *sk, struct ib_pool_fmr **_fmr, struct ib_umem **_umem) { - if (!sdp_sk(sk)->qp_active) + if (!sdp_sk(sk)->qp_active) { + sdp_warn(sk, "Trying to free fmr after destroying QP! fmr: %p\n", + *_fmr); return; + } ib_fmr_pool_unmap(*_fmr); *_fmr = NULL; @@ -571,7 +576,7 @@ int sdp_rdma_to_iovec(struct sock *sk, struct iovec *iov, struct sk_buff *skb, rc = sdp_alloc_fmr(sk, iov->iov_base, len, &rx_sa->fmr, &rx_sa->umem); if (rc) { - sdp_warn(sk, "Error allocating fmr: %d\n", rc); + sdp_dbg_data(sk, "Error allocating fmr: %d\n", rc); goto err_alloc_fmr; } @@ -610,7 +615,7 @@ err_post_send: err_alloc_fmr: if (rc && ssk->qp_active) { - sdp_warn(sk, "Couldn't do RDMA - post sendsm\n"); + sdp_dbg_data(sk, "Couldn't do RDMA - post sendsm\n"); rx_sa->flags |= RX_SA_ABORTED; } @@ -650,7 +655,7 @@ static int do_sdp_sendmsg_zcopy(struct sock *sk, struct tx_srcavail_state *tx_sa rc = sdp_alloc_fmr(sk, iov->iov_base, iov->iov_len, &tx_sa->fmr, &tx_sa->umem); if (rc) { - sdp_warn(sk, "Error allocating fmr: %d\n", rc); + sdp_dbg_data(sk, "Error allocating fmr: %d\n", rc); goto err_alloc_fmr; }