nvmet-tcp: Fix a possible sporadic response drops in weakly ordered arch

The order in which queue->cmd and rcv_state are updated is crucial.
If these assignments are reordered by the compiler, the worker might not
get queued in nvmet_tcp_queue_response(), hanging the IO. to enforce the
the correct reordering, set rcv_state using smp_store_release().

Fixes: bdaf13279192 ("nvmet-tcp: fix a segmentation fault during io parsing error")
Signed-off-by: Meir Elisha <meir.elisha@volumez.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Keith Busch <kbusch@kernel.org>

diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 7c51c2a8c109a9c14187d8717349f65a07825bea..4f9cac8a5abe07b2010bd3a9ba1dab83b5421514 100644 (file)
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -571,10 +571,16 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
        struct nvmet_tcp_cmd *cmd =
                container_of(req, struct nvmet_tcp_cmd, req);
        struct nvmet_tcp_queue  *queue = cmd->queue;
+       enum nvmet_tcp_recv_state queue_state;
+       struct nvmet_tcp_cmd *queue_cmd;
        struct nvme_sgl_desc *sgl;
        u32 len;
 
-       if (unlikely(cmd == queue->cmd)) {
+       /* Pairs with store_release in nvmet_prepare_receive_pdu() */
+       queue_state = smp_load_acquire(&queue->rcv_state);
+       queue_cmd = READ_ONCE(queue->cmd);
+
+       if (unlikely(cmd == queue_cmd)) {
                sgl = &cmd->req.cmd->common.dptr.sgl;
                len = le32_to_cpu(sgl->length);
 
@@ -583,7 +589,7 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
                 * Avoid using helpers, this might happen before
                 * nvmet_req_init is completed.
                 */
-               if (queue->rcv_state == NVMET_TCP_RECV_PDU &&
+               if (queue_state == NVMET_TCP_RECV_PDU &&
                    len && len <= cmd->req.port->inline_data_size &&
                    nvme_is_write(cmd->req.cmd))
                        return;
@@ -847,8 +853,9 @@ static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue)
 {
        queue->offset = 0;
        queue->left = sizeof(struct nvme_tcp_hdr);
-       queue->cmd = NULL;
-       queue->rcv_state = NVMET_TCP_RECV_PDU;
+       WRITE_ONCE(queue->cmd, NULL);
+       /* Ensure rcv_state is visible only after queue->cmd is set */
+       smp_store_release(&queue->rcv_state, NVMET_TCP_RECV_PDU);
 }
 
 static void nvmet_tcp_free_crypto(struct nvmet_tcp_queue *queue)