rcutorture: Add ability to test srcu_read_{,un}lock_fast()

This commit permits rcutorture to test srcu_read_{,un}lock_fast(), which
is specified by the rcutorture.reader_flavor=0x8 kernel boot parameter.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 1d2de50fb5d608a1685dbac81743001e95d7f1d3..1bd3eaa0b8e7ac258273241313a6a90406faa470 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -677,6 +677,7 @@ static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
 static int srcu_torture_read_lock(void)
 {
        int idx;
+       struct srcu_ctr __percpu *scp;
        int ret = 0;
 
        if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {
@@ -694,6 +695,12 @@ static int srcu_torture_read_lock(void)
                WARN_ON_ONCE(idx & ~0x1);
                ret += idx << 2;
        }
+       if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
+               scp = srcu_read_lock_fast(srcu_ctlp);
+               idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
+               WARN_ON_ONCE(idx & ~0x1);
+               ret += idx << 3;
+       }
        return ret;
 }
 
@@ -719,6 +726,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 static void srcu_torture_read_unlock(int idx)
 {
        WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
+       if (reader_flavor & SRCU_READ_FLAVOR_FAST)
+               srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
        if (reader_flavor & SRCU_READ_FLAVOR_LITE)
                srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2);
        if (reader_flavor & SRCU_READ_FLAVOR_NMI)