seqlock: Introduce raw_read_seqcount_latch()

Because with latches there is a strict data dependency on the seq load
we can avoid the rmb in favour of a read_barrier_depends.

Suggested-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 1c0cf3102fdc327a4474d51f6da13d9ee80b1b20..890c7ef709d5f3cc83873a1492cbdcebad6c0926 100644 (file)
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -35,6 +35,7 @@
 #include <linux/spinlock.h>
 #include <linux/preempt.h>
 #include <linux/lockdep.h>
+#include <linux/compiler.h>
 #include <asm/processor.h>
 
 /*
@@ -233,6 +234,11 @@ static inline void raw_write_seqcount_end(seqcount_t *s)
        s->sequence++;
 }
 
+static inline int raw_read_seqcount_latch(seqcount_t *s)
+{
+       return lockless_dereference(s->sequence);
+}
+
 /**
  * raw_write_seqcount_latch - redirect readers to even/odd copy
  * @s: pointer to seqcount_t
@@ -284,8 +290,7 @@ static inline void raw_write_seqcount_end(seqcount_t *s)
  *     unsigned seq, idx;
  *
  *     do {
- *             seq = latch->seq;
- *             smp_rmb();
+ *             seq = lockless_dereference(latch->seq);
  *
  *             idx = seq & 0x01;
  *             entry = data_query(latch->data[idx], ...);

diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index cbfedddbf0cb07be2c474e3881777ca033580415..266dafe8f015a8f15edfa00ac014a924d575fcd4 100644 (file)
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -393,7 +393,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
        u64 now;
 
        do {
-               seq = raw_read_seqcount(&tkf->seq);
+               seq = raw_read_seqcount_latch(&tkf->seq);
                tkr = tkf->base + (seq & 0x01);
                now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
        } while (read_seqcount_retry(&tkf->seq, seq));