SUNRPC: Use RMW bitops in single-threaded hot paths

I noticed CPU pipeline stalls while using perf.

Once an svc thread is scheduled and executing an RPC, no other
processes will touch svc_rqst::rq_flags. Thus bus-locked atomics are
not needed outside the svc thread scheduler.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 9dbce52f0f331f83d1d327e8a005cadae5c963bb..3895eb52d2b104494fdc23df4846e6863451f685 100644 (file)
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -970,7 +970,7 @@ nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         * the client wants us to do more in this compound:
         */
        if (!nfsd4_last_compound_op(rqstp))
-               clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
+               __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 
        /* check stateid */
        status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
@@ -2650,11 +2650,12 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
        cstate->minorversion = args->minorversion;
        fh_init(current_fh, NFS4_FHSIZE);
        fh_init(save_fh, NFS4_FHSIZE);
+
        /*
         * Don't use the deferral mechanism for NFSv4; compounds make it
         * too hard to avoid non-idempotency problems.
         */
-       clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
+       __clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
 
        /*
         * According to RFC3010, this takes precedence over all other errors.
@@ -2769,7 +2770,7 @@ encode_op:
 out:
        cstate->status = status;
        /* Reset deferral mechanism for RPC deferrals */
-       set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
+       __set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
        return rpc_success;
 }
 

diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index da92e7d2ab6a1157b8eee56516f603c9bfd08450..61b2aae81abb750196299772d08d46e5daca021f 100644 (file)
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -2411,7 +2411,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
        argp->rqstp->rq_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE;
 
        if (readcount > 1 || max_reply > PAGE_SIZE - auth_slack)
-               clear_bit(RQ_SPLICE_OK, &argp->rqstp->rq_flags);
+               __clear_bit(RQ_SPLICE_OK, &argp->rqstp->rq_flags);
 
        return true;
 }

diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index c2ba9d4cd2c777a212c45a0c51fd541214597a06..bcd74dddbe2db88bbe36670a291729250e8f64d7 100644 (file)
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -900,7 +900,7 @@ unwrap_integ_data(struct svc_rqst *rqstp, struct xdr_buf *buf, u32 seq, struct g
         * rejecting the server-computed MIC in this somewhat rare case,
         * do not use splice with the GSS integrity service.
         */
-       clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
+       __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 
        /* Did we already verify the signature on the original pass through? */
        if (rqstp->rq_deferred)
@@ -972,7 +972,7 @@ unwrap_priv_data(struct svc_rqst *rqstp, struct xdr_buf *buf, u32 seq, struct gs
        int pad, remaining_len, offset;
        u32 rseqno;
 
-       clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
+       __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 
        priv_len = svc_getnl(&buf->head[0]);
        if (rqstp->rq_deferred) {

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 56bed0a57eea32e5eb796d1bdd680a64dac47ede..7c9a0d0b12300425a4a707f8469b6333446a26b8 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1244,10 +1244,10 @@ svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
                goto err_short_len;
 
        /* Will be turned off by GSS integrity and privacy services */
-       set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
+       __set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
        /* Will be turned off only when NFSv4 Sessions are used */
-       set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
-       clear_bit(RQ_DROPME, &rqstp->rq_flags);
+       __set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
+       __clear_bit(RQ_DROPME, &rqstp->rq_flags);
 
        svc_putu32(resv, rqstp->rq_xid);
 

diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index 5df90bca10edaa4848e43ac22045f7ffd164cd1c..2c4dd7ca95b0cd6eedb9919dda1f3b751130b5a9 100644 (file)
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -1238,7 +1238,7 @@ static struct cache_deferred_req *svc_defer(struct cache_req *req)
        trace_svc_defer(rqstp);
        svc_xprt_get(rqstp->rq_xprt);
        dr->xprt = rqstp->rq_xprt;
-       set_bit(RQ_DROPME, &rqstp->rq_flags);
+       __set_bit(RQ_DROPME, &rqstp->rq_flags);
 
        dr->handle.revisit = svc_revisit;
        return &dr->handle;

diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index 05452318afecd7e1502b0917637785297eb83288..b3c9740cfd358df2426870b7fcb05943b06eb73c 100644 (file)
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -298,9 +298,9 @@ static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs)
 static void svc_sock_secure_port(struct svc_rqst *rqstp)
 {
        if (svc_port_is_privileged(svc_addr(rqstp)))
-               set_bit(RQ_SECURE, &rqstp->rq_flags);
+               __set_bit(RQ_SECURE, &rqstp->rq_flags);
        else
-               clear_bit(RQ_SECURE, &rqstp->rq_flags);
+               __clear_bit(RQ_SECURE, &rqstp->rq_flags);
 }
 
 /*
@@ -1008,9 +1008,9 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
        rqstp->rq_xprt_ctxt   = NULL;
        rqstp->rq_prot        = IPPROTO_TCP;
        if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))
-               set_bit(RQ_LOCAL, &rqstp->rq_flags);
+               __set_bit(RQ_LOCAL, &rqstp->rq_flags);
        else
-               clear_bit(RQ_LOCAL, &rqstp->rq_flags);
+               __clear_bit(RQ_LOCAL, &rqstp->rq_flags);
 
        p = (__be32 *)rqstp->rq_arg.head[0].iov_base;
        calldir = p[1];

diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 94b20fb471356356885da1b482750f387c82c9ab..199fa012f18a879a6d3389dfc3b125cc12d14678 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -602,7 +602,7 @@ static int svc_rdma_has_wspace(struct svc_xprt *xprt)
 
 static void svc_rdma_secure_port(struct svc_rqst *rqstp)
 {
-       set_bit(RQ_SECURE, &rqstp->rq_flags);
+       __set_bit(RQ_SECURE, &rqstp->rq_flags);
 }
 
 static void svc_rdma_kill_temp_xprt(struct svc_xprt *xprt)