nvme: properly free resources for cancelled command

We need to move freeing of resources to the ->complete handler to ensure
they are also freed when we cancel the command.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Keith Busch <keith.busch@intel.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
(cherry picked from commit eee417b0697827a6e120199b126b447af3c81b47)

Orabug: 25130845

Signed-off-by: Ashok Vairavan <ashok.vairavan@oracle.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 2317edae252849315772d71d2bd55e5ace1f546e..e080e03dfe5bf495d29c0ef3bf0ae352651f08cd 100644 (file)
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -89,12 +89,12 @@ static wait_queue_head_t nvme_kthread_wait;
 
 struct nvme_dev;
 struct nvme_queue;
-struct nvme_iod;
 
 static int nvme_reset(struct nvme_dev *dev);
 static int nvme_process_cq(struct nvme_queue *nvmeq);
 static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod);
 static void nvme_dead_ctrl(struct nvme_dev *dev);
+static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
 static void nvme_dev_shutdown(struct nvme_dev *dev);
 
 struct async_cmd_info {
@@ -569,42 +569,6 @@ static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 }
 #endif
 
-static void req_completion(struct nvme_queue *nvmeq, struct nvme_completion *cqe)
-{
-       struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
-       struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
-       struct nvme_iod *iod = cmd_rq->iod;
-       u16 status = le16_to_cpup(&cqe->status) >> 1;
-       int error = 0;
-
-       if (unlikely(status)) {
-               if (nvme_req_needs_retry(req, status)) {
-                       nvme_unmap_data(nvmeq->dev, iod);
-                       nvme_requeue_req(req);
-                       return;
-               }
-
-               if (req->cmd_type == REQ_TYPE_DRV_PRIV)
-                       error = status;
-               else
-                       error = nvme_error_status(status);
-       }
-
-       if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
-               u32 result = le32_to_cpup(&cqe->result);
-               req->special = (void *)(uintptr_t)result;
-       }
-
-       if (cmd_rq->aborted)
-               dev_warn(nvmeq->dev->dev,
-                       "completing aborted command with status:%04x\n",
-                       status);
-
-       nvme_unmap_data(nvmeq->dev, iod);
-       blk_mq_complete_request(req);
-}
-
-/* length is in bytes.  gfp flags indicates whether we may sleep. */
 static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
                int total_len)
 {
@@ -806,7 +770,7 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
                if (!(ns->pi_type && ns->ms == 8) &&
                                        req->cmd_type != REQ_TYPE_DRV_PRIV) {
                        req->errors = -EFAULT;
-                       blk_mq_complete_request(req);
+                       blk_mq_end_request(req, -EFAULT);
                        return BLK_MQ_RQ_QUEUE_OK;
                }
        }
@@ -847,6 +811,35 @@ out:
        return ret;
 }
 
+static void nvme_complete_rq(struct request *req)
+{
+       struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
+       struct nvme_dev *dev = cmd->nvmeq->dev;
+       int error = 0;
+
+       nvme_unmap_data(dev, cmd->iod);
+
+       if (unlikely(req->errors)) {
+               if (nvme_req_needs_retry(req, req->errors)) {
+                       nvme_requeue_req(req);
+                       return;
+               }
+
+               if (req->cmd_type == REQ_TYPE_DRV_PRIV)
+                       error = req->errors;
+               else
+                       error = nvme_error_status(req->errors);
+       }
+
+       if (unlikely(cmd->aborted)) {
+               dev_warn(dev->dev,
+                       "completing aborted command with status: %04x\n",
+                       req->errors);
+       }
+
+       blk_mq_end_request(req, error);
+}
+
 static int nvme_process_cq(struct nvme_queue *nvmeq)
 {
        u16 head, phase;
@@ -857,6 +850,7 @@ static int nvme_process_cq(struct nvme_queue *nvmeq)
        for (;;) {
                struct nvme_completion cqe = nvmeq->cqes[head];
                u16 status = le16_to_cpu(cqe.status);
+               struct request *req;
 
                if ((status & 1) != phase)
                        break;
@@ -885,7 +879,13 @@ static int nvme_process_cq(struct nvme_queue *nvmeq)
                        continue;
                }
 
-               req_completion(nvmeq, &cqe);
+               req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
+               if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
+                       u32 result = le32_to_cpu(cqe.result);
+                       req->special = (void *)(uintptr_t)result;
+               }
+               blk_mq_complete_request(req, status >> 1);
+
        }
 
        /* If the controller ignores the cq head doorbell and continuously
@@ -1336,6 +1336,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 
 static struct blk_mq_ops nvme_mq_admin_ops = {
        .queue_rq       = nvme_queue_rq,
+       .complete       = nvme_complete_rq,
        .map_queue      = blk_mq_map_queue,
        .init_hctx      = nvme_admin_init_hctx,
        .exit_hctx      = nvme_admin_exit_hctx,
@@ -1345,6 +1346,7 @@ static struct blk_mq_ops nvme_mq_admin_ops = {
 
 static struct blk_mq_ops nvme_mq_ops = {
        .queue_rq       = nvme_queue_rq,
+       .complete       = nvme_complete_rq,
        .map_queue      = blk_mq_map_queue,
        .init_hctx      = nvme_init_hctx,
        .init_request   = nvme_init_request,