sdp: make statistics per cpu

- Caused lots of cache misses during multi stream traffic

Signed-off-by: Amir Vadai <amirv@mellanox.co.il>

diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h
index 6a648c7afb285f6db7ee37dabb6391f9a5b6f143..dc5332c2d6a3d4889a717d42824733f48884f1bd 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp.h
+++ b/drivers/infiniband/ulp/sdp/sdp.h
@@ -102,7 +102,9 @@ extern struct workqueue_struct *rx_comp_wq;
 extern atomic_t sdp_current_mem_usage;
 extern spinlock_t sdp_large_sockets_lock;
 extern struct ib_client sdp_client;
-extern struct sdpstats sdpstats;
+#ifdef SDPSTATS_ON
+DECLARE_PER_CPU(struct sdpstats, sdpstats);
+#endif
 
 enum sdp_mid {
        SDP_MID_HELLO = 0x0,
@@ -672,6 +674,7 @@ static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk)
 
 #ifdef SDPSTATS_ON
 
+#define SDPSTATS_MAX_HIST_SIZE 256
 struct sdpstats {
        u32 post_send[256];
        u32 sendmsg_bcopy_segment;
@@ -707,15 +710,15 @@ static inline void sdpstats_hist(u32 *h, u32 val, u32 maxidx, int is_log)
        h[idx]++;
 }
 
-#define SDPSTATS_COUNTER_INC(stat) do { sdpstats.stat++; } while (0)
-#define SDPSTATS_COUNTER_ADD(stat, val) do { sdpstats.stat += val; } while (0)
-#define SDPSTATS_COUNTER_MID_INC(stat, mid) do { sdpstats.stat[mid]++; } \
+#define SDPSTATS_COUNTER_INC(stat) do { __get_cpu_var(sdpstats).stat++; } while (0)
+#define SDPSTATS_COUNTER_ADD(stat, val) do { __get_cpu_var(sdpstats).stat += val; } while (0)
+#define SDPSTATS_COUNTER_MID_INC(stat, mid) do { __get_cpu_var(sdpstats).stat[mid]++; } \
        while (0)
 #define SDPSTATS_HIST(stat, size) \
-       sdpstats_hist(sdpstats.stat, size, ARRAY_SIZE(sdpstats.stat) - 1, 1)
+       sdpstats_hist(__get_cpu_var(sdpstats).stat, size, ARRAY_SIZE(__get_cpu_var(sdpstats).stat) - 1, 1)
 
 #define SDPSTATS_HIST_LINEAR(stat, size) \
-       sdpstats_hist(sdpstats.stat, size, ARRAY_SIZE(sdpstats.stat) - 1, 0)
+       sdpstats_hist(__get_cpu_var(sdpstats).stat, size, ARRAY_SIZE(__get_cpu_var(sdpstats).stat) - 1, 0)
 
 #else
 #define SDPSTATS_COUNTER_INC(stat)

diff --git a/drivers/infiniband/ulp/sdp/sdp_proc.c b/drivers/infiniband/ulp/sdp/sdp_proc.c
index 290b1edc9a3d1fceb69af2d14f1c83f965fe7f11..f7cef1ff1202677c9ac6801fca1a319a3a1060e4 100644 (file)
--- a/drivers/infiniband/ulp/sdp/sdp_proc.c
+++ b/drivers/infiniband/ulp/sdp/sdp_proc.c
@@ -202,7 +202,7 @@ static struct sdp_seq_afinfo sdp_seq_afinfo = {
 };
 
 #ifdef SDPSTATS_ON
-struct sdpstats sdpstats = { { 0 } };
+DEFINE_PER_CPU(struct sdpstats, sdpstats);
 
 static void sdpstats_seq_hist(struct seq_file *seq, char *str, u32 *h, int n,
                int is_log)
@@ -233,69 +233,97 @@ static void sdpstats_seq_hist(struct seq_file *seq, char *str, u32 *h, int n,
        }
 }
 
+#define SDPSTATS_COUNTER_GET(var) ({ \
+       u32 __val = 0;                                          \
+       unsigned int __i;                                       \
+       for_each_possible_cpu(__i)                              \
+               __val += per_cpu(sdpstats, __i).var;            \
+       __val;                                                  \
+})     
+
+#define SDPSTATS_HIST_GET(hist, hist_len, sum) ({ \
+       unsigned int __i;                                       \
+       for_each_possible_cpu(__i) {                            \
+               unsigned int __j;                               \
+               u32 *h = per_cpu(sdpstats, __i).hist;           \
+               for (__j = 0; __j < hist_len; __j++) {          \
+                       sum[__j] += h[__j];                     \
+               } \
+       }                                                       \
+})
+
+#define __sdpstats_seq_hist(seq, msg, hist, is_log) ({         \
+       u32 tmp_hist[SDPSTATS_MAX_HIST_SIZE];                   \
+       int hist_len = ARRAY_SIZE(__get_cpu_var(sdpstats).hist);\
+       memset(tmp_hist, 0, sizeof(tmp_hist));                  \
+       SDPSTATS_HIST_GET(hist, hist_len, tmp_hist);    \
+       sdpstats_seq_hist(seq, msg, tmp_hist, hist_len, is_log);\
+})
+
 static int sdpstats_seq_show(struct seq_file *seq, void *v)
 {
        int i;
 
        seq_printf(seq, "SDP statistics:\n");
 
-       sdpstats_seq_hist(seq, "sendmsg_seglen", sdpstats.sendmsg_seglen,
-               ARRAY_SIZE(sdpstats.sendmsg_seglen), 1);
-
-       sdpstats_seq_hist(seq, "send_size", sdpstats.send_size,
-               ARRAY_SIZE(sdpstats.send_size), 1);
+       __sdpstats_seq_hist(seq, "sendmsg_seglen", sendmsg_seglen, 1);
+       __sdpstats_seq_hist(seq, "send_size", send_size, 1);
+       __sdpstats_seq_hist(seq, "credits_before_update",
+               credits_before_update, 0);
 
-       sdpstats_seq_hist(seq, "credits_before_update",
-               sdpstats.credits_before_update,
-               ARRAY_SIZE(sdpstats.credits_before_update), 0);
-
-       seq_printf(seq, "sdp_sendmsg() calls\t\t: %d\n", sdpstats.sendmsg);
+       seq_printf(seq, "sdp_sendmsg() calls\t\t: %d\n",
+               SDPSTATS_COUNTER_GET(sendmsg));
        seq_printf(seq, "bcopy segments     \t\t: %d\n",
-               sdpstats.sendmsg_bcopy_segment);
+               SDPSTATS_COUNTER_GET(sendmsg_bcopy_segment));
        seq_printf(seq, "bzcopy segments    \t\t: %d\n",
-               sdpstats.sendmsg_bzcopy_segment);
+               SDPSTATS_COUNTER_GET(sendmsg_bzcopy_segment));
        seq_printf(seq, "zcopy segments    \t\t: %d\n",
-               sdpstats.sendmsg_zcopy_segment);
+               SDPSTATS_COUNTER_GET(sendmsg_zcopy_segment));
        seq_printf(seq, "post_send_credits  \t\t: %d\n",
-               sdpstats.post_send_credits);
+               SDPSTATS_COUNTER_GET(post_send_credits));
        seq_printf(seq, "memcpy_count       \t\t: %u\n",
-               sdpstats.memcpy_count);
+               SDPSTATS_COUNTER_GET(memcpy_count));
 
-       for (i = 0; i < ARRAY_SIZE(sdpstats.post_send); i++) {
-               if (mid2str(i)) {
-                       seq_printf(seq, "post_send %-20s\t: %d\n",
-                                       mid2str(i), sdpstats.post_send[i]);
-               }
-       }
+        for (i = 0; i < ARRAY_SIZE(__get_cpu_var(sdpstats).post_send); i++) {
+                if (mid2str(i)) {
+                        seq_printf(seq, "post_send %-20s\t: %d\n",
+                                        mid2str(i),
+                                       SDPSTATS_COUNTER_GET(post_send[i]));
+                }
+        }
 
        seq_printf(seq, "\n");
-       seq_printf(seq, "post_recv         \t\t: %d\n", sdpstats.post_recv);
+       seq_printf(seq, "post_recv         \t\t: %d\n",
+               SDPSTATS_COUNTER_GET(post_recv));
        seq_printf(seq, "BZCopy poll miss  \t\t: %d\n",
-               sdpstats.bzcopy_poll_miss);
+               SDPSTATS_COUNTER_GET(bzcopy_poll_miss));
        seq_printf(seq, "send_wait_for_mem \t\t: %d\n",
-               sdpstats.send_wait_for_mem);
+               SDPSTATS_COUNTER_GET(send_wait_for_mem));
        seq_printf(seq, "send_miss_no_credits\t\t: %d\n",
-               sdpstats.send_miss_no_credits);
+               SDPSTATS_COUNTER_GET(send_miss_no_credits));
 
-       seq_printf(seq, "rx_poll_miss      \t\t: %d\n", sdpstats.rx_poll_miss);
-       seq_printf(seq, "tx_poll_miss      \t\t: %d\n", sdpstats.tx_poll_miss);
-       seq_printf(seq, "tx_poll_busy      \t\t: %d\n", sdpstats.tx_poll_busy);
-       seq_printf(seq, "tx_poll_hit       \t\t: %d\n", sdpstats.tx_poll_hit);
+       seq_printf(seq, "rx_poll_miss      \t\t: %d\n", SDPSTATS_COUNTER_GET(rx_poll_miss));
+       seq_printf(seq, "tx_poll_miss      \t\t: %d\n", SDPSTATS_COUNTER_GET(tx_poll_miss));
+       seq_printf(seq, "tx_poll_busy      \t\t: %d\n", SDPSTATS_COUNTER_GET(tx_poll_busy));
+       seq_printf(seq, "tx_poll_hit       \t\t: %d\n", SDPSTATS_COUNTER_GET(tx_poll_hit));
 
        seq_printf(seq, "CQ stats:\n");
-       seq_printf(seq, "- RX interrupts\t\t: %d\n", sdpstats.rx_int_count);
-       seq_printf(seq, "- TX interrupts\t\t: %d\n", sdpstats.tx_int_count);
+       seq_printf(seq, "- RX interrupts\t\t: %d\n", SDPSTATS_COUNTER_GET(rx_int_count));
+       seq_printf(seq, "- TX interrupts\t\t: %d\n", SDPSTATS_COUNTER_GET(tx_int_count));
 
        seq_printf(seq, "ZCopy stats:\n");
-       seq_printf(seq, "- TX timeout\t\t: %d\n", sdpstats.zcopy_tx_timeout);
-       seq_printf(seq, "- TX error\t\t: %d\n", sdpstats.zcopy_tx_error);
+       seq_printf(seq, "- TX timeout\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_timeout));
+       seq_printf(seq, "- TX error\t\t: %d\n", SDPSTATS_COUNTER_GET(zcopy_tx_error));
        return 0;
 }
 
 static ssize_t sdpstats_write(struct file *file, const char __user *buf,
                            size_t count, loff_t *offs)
 {
-       memset(&sdpstats, 0, sizeof(sdpstats));
+       int i;
+
+       for_each_possible_cpu(i)
+               memset(&per_cpu(sdpstats, i), 0, sizeof(struct sdpstats));
        printk(KERN_WARNING "Cleared sdp statistics\n");
 
        return count;