bpf: Use u64_stats_t in struct bpf_prog_stats

Commit 316580b69d0a ("u64_stats: provide u64_stats_t type")
fixed possible load/store tearing on 64bit arches.

For instance the following C code

stats->nsecs += sched_clock() - start;

Could be rightfully implemented like this by a compiler,
confusing concurrent readers a lot:

stats->nsecs += sched_clock();
// arbitrary delay
stats->nsecs -= start;

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20211026214133.3114279-4-eric.dumazet@gmail.com

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 2fffe9cc50f946f24f4f78b59902bad91f910e5b..9782e32458522273b314579e62c6215ebb07a617 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -553,9 +553,9 @@ struct bpf_binary_header {
 };
 
 struct bpf_prog_stats {
-       u64 cnt;
-       u64 nsecs;
-       u64 misses;
+       u64_stats_t cnt;
+       u64_stats_t nsecs;
+       u64_stats_t misses;
        struct u64_stats_sync syncp;
 } __aligned(2 * sizeof(u64));
 
@@ -617,8 +617,8 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog,
                ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
                stats = this_cpu_ptr(prog->stats);
                flags = u64_stats_update_begin_irqsave(&stats->syncp);
-               stats->cnt++;
-               stats->nsecs += sched_clock() - start;
+               u64_stats_inc(&stats->cnt);
+               u64_stats_add(&stats->nsecs, sched_clock() - start);
                u64_stats_update_end_irqrestore(&stats->syncp, flags);
        } else {
                ret = dfunc(ctx, prog->insnsi, prog->bpf_func);

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 5beb321b3b3b66765b6592a0f53336ee9cbf737c..3e1c024ce3ed14673e5021ae6e7226c02b0393fa 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1804,8 +1804,14 @@ static int bpf_prog_release(struct inode *inode, struct file *filp)
        return 0;
 }
 
+struct bpf_prog_kstats {
+       u64 nsecs;
+       u64 cnt;
+       u64 misses;
+};
+
 static void bpf_prog_get_stats(const struct bpf_prog *prog,
-                              struct bpf_prog_stats *stats)
+                              struct bpf_prog_kstats *stats)
 {
        u64 nsecs = 0, cnt = 0, misses = 0;
        int cpu;
@@ -1818,9 +1824,9 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog,
                st = per_cpu_ptr(prog->stats, cpu);
                do {
                        start = u64_stats_fetch_begin_irq(&st->syncp);
-                       tnsecs = st->nsecs;
-                       tcnt = st->cnt;
-                       tmisses = st->misses;
+                       tnsecs = u64_stats_read(&st->nsecs);
+                       tcnt = u64_stats_read(&st->cnt);
+                       tmisses = u64_stats_read(&st->misses);
                } while (u64_stats_fetch_retry_irq(&st->syncp, start));
                nsecs += tnsecs;
                cnt += tcnt;
@@ -1836,7 +1842,7 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 {
        const struct bpf_prog *prog = filp->private_data;
        char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
-       struct bpf_prog_stats stats;
+       struct bpf_prog_kstats stats;
 
        bpf_prog_get_stats(prog, &stats);
        bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
@@ -3577,7 +3583,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,
        struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
        struct bpf_prog_info info;
        u32 info_len = attr->info.info_len;
-       struct bpf_prog_stats stats;
+       struct bpf_prog_kstats stats;
        char __user *uinsns;
        u32 ulen;
        int err;

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index e5963de368ed34874414d097bc6614ff7e168dd3..e98de5e73ba59f756480cc5f962849be1859751b 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -545,7 +545,7 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)
 
        stats = this_cpu_ptr(prog->stats);
        u64_stats_update_begin(&stats->syncp);
-       stats->misses++;
+       u64_stats_inc(&stats->misses);
        u64_stats_update_end(&stats->syncp);
 }
 
@@ -590,8 +590,8 @@ static void notrace update_prog_stats(struct bpf_prog *prog,
 
                stats = this_cpu_ptr(prog->stats);
                flags = u64_stats_update_begin_irqsave(&stats->syncp);
-               stats->cnt++;
-               stats->nsecs += sched_clock() - start;
+               u64_stats_inc(&stats->cnt);
+               u64_stats_add(&stats->nsecs, sched_clock() - start);
                u64_stats_update_end_irqrestore(&stats->syncp, flags);
        }
 }