vfio/mlx5: Rewrite create mkey flow to allow better code reuse

Change the creation of mkey to be performed in multiple steps:
data allocation, DMA setup and actual call to HW to create that mkey.

In this new flow, the whole input to MKEY command is saved to eliminate
the need to keep array of pointers for DMA addresses for receive list
and in the future patches for send list too.

In addition to memory size reduce and elimination of unnecessary data
movements to set MKEY input, the code is prepared for future reuse.

Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c
index fdc3e515741fc15d00c92764a52ed75789ffe00d..1832a6c1f35d3e1f1805f53bad2902d0bb6831d5 100644 (file)
--- a/drivers/vfio/pci/mlx5/cmd.c
+++ b/drivers/vfio/pci/mlx5/cmd.c
@@ -313,39 +313,21 @@ err_exec:
        return ret;
 }
 
-static int _create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
-                       struct mlx5_vhca_data_buffer *buf,
-                       struct mlx5_vhca_recv_buf *recv_buf,
-                       u32 *mkey)
+static u32 *alloc_mkey_in(u32 npages, u32 pdn)
 {
-       size_t npages = buf ? buf->npages : recv_buf->npages;
-       int err = 0, inlen;
-       __be64 *mtt;
+       int inlen;
        void *mkc;
        u32 *in;
 
        inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
-               sizeof(*mtt) * round_up(npages, 2);
+               sizeof(__be64) * round_up(npages, 2);
 
-       in = kvzalloc(inlen, GFP_KERNEL);
+       in = kvzalloc(inlen, GFP_KERNEL_ACCOUNT);
        if (!in)
-               return -ENOMEM;
+               return NULL;
 
        MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
                 DIV_ROUND_UP(npages, 2));
-       mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
-
-       if (buf) {
-               struct sg_dma_page_iter dma_iter;
-
-               for_each_sgtable_dma_page(&buf->table.sgt, &dma_iter, 0)
-                       *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter));
-       } else {
-               int i;
-
-               for (i = 0; i < npages; i++)
-                       *mtt++ = cpu_to_be64(recv_buf->dma_addrs[i]);
-       }
 
        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
        MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
@@ -359,9 +341,29 @@ static int _create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
        MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
        MLX5_SET(mkc, mkc, translations_octword_size, DIV_ROUND_UP(npages, 2));
        MLX5_SET64(mkc, mkc, len, npages * PAGE_SIZE);
-       err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
-       kvfree(in);
-       return err;
+
+       return in;
+}
+
+static int create_mkey(struct mlx5_core_dev *mdev, u32 npages,
+                      struct mlx5_vhca_data_buffer *buf, u32 *mkey_in,
+                      u32 *mkey)
+{
+       __be64 *mtt;
+       int inlen;
+
+       mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, mkey_in, klm_pas_mtt);
+       if (buf) {
+               struct sg_dma_page_iter dma_iter;
+
+               for_each_sgtable_dma_page(&buf->table.sgt, &dma_iter, 0)
+                       *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter));
+       }
+
+       inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
+               sizeof(__be64) * round_up(npages, 2);
+
+       return mlx5_core_create_mkey(mdev, mkey, mkey_in, inlen);
 }
 
 static int mlx5vf_dma_data_buffer(struct mlx5_vhca_data_buffer *buf)
@@ -374,20 +376,28 @@ static int mlx5vf_dma_data_buffer(struct mlx5_vhca_data_buffer *buf)
        if (mvdev->mdev_detach)
                return -ENOTCONN;
 
-       if (buf->dmaed || !buf->npages)
+       if (buf->mkey_in || !buf->npages)
                return -EINVAL;
 
        ret = dma_map_sgtable(mdev->device, &buf->table.sgt, buf->dma_dir, 0);
        if (ret)
                return ret;
 
-       ret = _create_mkey(mdev, buf->migf->pdn, buf, NULL, &buf->mkey);
-       if (ret)
+       buf->mkey_in = alloc_mkey_in(buf->npages, buf->migf->pdn);
+       if (!buf->mkey_in) {
+               ret = -ENOMEM;
                goto err;
+       }
 
-       buf->dmaed = true;
+       ret = create_mkey(mdev, buf->npages, buf, buf->mkey_in, &buf->mkey);
+       if (ret)
+               goto err_create_mkey;
 
        return 0;
+
+err_create_mkey:
+       kvfree(buf->mkey_in);
+       buf->mkey_in = NULL;
 err:
        dma_unmap_sgtable(mdev->device, &buf->table.sgt, buf->dma_dir, 0);
        return ret;
@@ -401,8 +411,9 @@ void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf)
        lockdep_assert_held(&migf->mvdev->state_mutex);
        WARN_ON(migf->mvdev->mdev_detach);
 
-       if (buf->dmaed) {
+       if (buf->mkey_in) {
                mlx5_core_destroy_mkey(migf->mvdev->mdev, buf->mkey);
+               kvfree(buf->mkey_in);
                dma_unmap_sgtable(migf->mvdev->mdev->device, &buf->table.sgt,
                                  buf->dma_dir, 0);
        }
@@ -779,7 +790,7 @@ int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
        if (mvdev->mdev_detach)
                return -ENOTCONN;
 
-       if (!buf->dmaed) {
+       if (!buf->mkey_in) {
                err = mlx5vf_dma_data_buffer(buf);
                if (err)
                        return err;
@@ -1380,56 +1391,54 @@ err:
        kvfree(recv_buf->page_list);
        return -ENOMEM;
 }
+static void unregister_dma_pages(struct mlx5_core_dev *mdev, u32 npages,
+                                u32 *mkey_in)
+{
+       dma_addr_t addr;
+       __be64 *mtt;
+       int i;
+
+       mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, mkey_in, klm_pas_mtt);
+       for (i = npages - 1; i >= 0; i--) {
+               addr = be64_to_cpu(mtt[i]);
+               dma_unmap_single(mdev->device, addr, PAGE_SIZE,
+                               DMA_FROM_DEVICE);
+       }
+}
 
-static int register_dma_recv_pages(struct mlx5_core_dev *mdev,
-                                  struct mlx5_vhca_recv_buf *recv_buf)
+static int register_dma_pages(struct mlx5_core_dev *mdev, u32 npages,
+                             struct page **page_list, u32 *mkey_in)
 {
-       int i, j;
+       dma_addr_t addr;
+       __be64 *mtt;
+       int i;
 
-       recv_buf->dma_addrs = kvcalloc(recv_buf->npages,
-                                      sizeof(*recv_buf->dma_addrs),
-                                      GFP_KERNEL_ACCOUNT);
-       if (!recv_buf->dma_addrs)
-               return -ENOMEM;
+       mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, mkey_in, klm_pas_mtt);
 
-       for (i = 0; i < recv_buf->npages; i++) {
-               recv_buf->dma_addrs[i] = dma_map_page(mdev->device,
-                                                     recv_buf->page_list[i],
-                                                     0, PAGE_SIZE,
-                                                     DMA_FROM_DEVICE);
-               if (dma_mapping_error(mdev->device, recv_buf->dma_addrs[i]))
+       for (i = 0; i < npages; i++) {
+               addr = dma_map_page(mdev->device, page_list[i], 0, PAGE_SIZE,
+                                   DMA_FROM_DEVICE);
+               if (dma_mapping_error(mdev->device, addr))
                        goto error;
+
+               *mtt++ = cpu_to_be64(addr);
        }
+
        return 0;
 
 error:
-       for (j = 0; j < i; j++)
-               dma_unmap_single(mdev->device, recv_buf->dma_addrs[j],
-                                PAGE_SIZE, DMA_FROM_DEVICE);
-
-       kvfree(recv_buf->dma_addrs);
+       unregister_dma_pages(mdev, i, mkey_in);
        return -ENOMEM;
 }
 
-static void unregister_dma_recv_pages(struct mlx5_core_dev *mdev,
-                                     struct mlx5_vhca_recv_buf *recv_buf)
-{
-       int i;
-
-       for (i = 0; i < recv_buf->npages; i++)
-               dma_unmap_single(mdev->device, recv_buf->dma_addrs[i],
-                                PAGE_SIZE, DMA_FROM_DEVICE);
-
-       kvfree(recv_buf->dma_addrs);
-}
-
 static void mlx5vf_free_qp_recv_resources(struct mlx5_core_dev *mdev,
                                          struct mlx5_vhca_qp *qp)
 {
        struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf;
 
        mlx5_core_destroy_mkey(mdev, recv_buf->mkey);
-       unregister_dma_recv_pages(mdev, recv_buf);
+       unregister_dma_pages(mdev, recv_buf->npages, recv_buf->mkey_in);
+       kvfree(recv_buf->mkey_in);
        free_recv_pages(&qp->recv_buf);
 }
 
@@ -1445,18 +1454,29 @@ static int mlx5vf_alloc_qp_recv_resources(struct mlx5_core_dev *mdev,
        if (err < 0)
                return err;
 
-       err = register_dma_recv_pages(mdev, recv_buf);
-       if (err)
+       recv_buf->mkey_in = alloc_mkey_in(npages, pdn);
+       if (!recv_buf->mkey_in) {
+               err = -ENOMEM;
                goto end;
+       }
+
+       err = register_dma_pages(mdev, npages, recv_buf->page_list,
+                                recv_buf->mkey_in);
+       if (err)
+               goto err_register_dma;
 
-       err = _create_mkey(mdev, pdn, NULL, recv_buf, &recv_buf->mkey);
+       err = create_mkey(mdev, npages, NULL, recv_buf->mkey_in,
+                         &recv_buf->mkey);
        if (err)
                goto err_create_mkey;
 
        return 0;
 
 err_create_mkey:
-       unregister_dma_recv_pages(mdev, recv_buf);
+       unregister_dma_pages(mdev, npages, recv_buf->mkey_in);
+err_register_dma:
+       kvfree(recv_buf->mkey_in);
+       recv_buf->mkey_in = NULL;
 end:
        free_recv_pages(recv_buf);
        return err;

diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h
index 7d4a833b6900ae69b8549823549f8dbe2a1fe84c..25dd6ff5459186f81d674e6fc3c013b16b95c773 100644 (file)
--- a/drivers/vfio/pci/mlx5/cmd.h
+++ b/drivers/vfio/pci/mlx5/cmd.h
@@ -58,8 +58,8 @@ struct mlx5_vhca_data_buffer {
        u64 length;
        u32 npages;
        u32 mkey;
+       u32 *mkey_in;
        enum dma_data_direction dma_dir;
-       u8 dmaed:1;
        u8 stop_copy_chunk_num;
        struct list_head buf_elm;
        struct mlx5_vf_migration_file *migf;
@@ -133,8 +133,8 @@ struct mlx5_vhca_cq {
 struct mlx5_vhca_recv_buf {
        u32 npages;
        struct page **page_list;
-       dma_addr_t *dma_addrs;
        u32 next_rq_offset;
+       u32 *mkey_in;
        u32 mkey;
 };