RDMA/mlx5: Zero out ODP related items in the mlx5_ib_mr

All of the ODP code assumes when it calls mlx5_mr_cache_alloc() the ODP
related fields are zero'd. This is true if the MR was just allocated, but
if the MR is recycled through the cache then the values are never zero'd.

This causes a bug in the odp_stats, they don't reset when the MR is
reallocated, also is_odp_implicit is never 0'd.

So we can use memset on a block of the mlx5_ib_mr reorganize the structure
to put all the data that can be zero'd by the cache at the end.

It is organized as an anonymous struct because the next patch will make
this a union.

Delete the unused smr_info. Don't set the kernel only desc_size on the
user path. No longer any need to zero mr->parent before freeing it, the
memset() will get it now.

Fixes: a3de94e3d61e ("IB/mlx5: Introduce ODP diagnostic counters")
Link: https://lore.kernel.org/r/20210304120745.1090751-2-leon@kernel.org
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 88cc26e008fce033da2d8553b683a871fab9f625..b085c02b53d0ea39565e02fd460f58cc5b7259e8 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -547,11 +547,6 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
        return container_of(wr, struct mlx5_umr_wr, wr);
 }
 
-struct mlx5_shared_mr_info {
-       int mr_id;
-       struct ib_umem          *umem;
-};
-
 enum mlx5_ib_cq_pr_flags {
        MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0,
 };
@@ -654,47 +649,69 @@ struct mlx5_ib_dm {
        atomic64_add(value, &((mr)->odp_stats.counter_name))
 
 struct mlx5_ib_mr {
-       struct ib_mr            ibmr;
-       void                    *descs;
-       dma_addr_t              desc_map;
-       int                     ndescs;
-       int                     data_length;
-       int                     meta_ndescs;
-       int                     meta_length;
-       int                     max_descs;
-       int                     desc_size;
-       int                     access_mode;
-       unsigned int            page_shift;
-       struct mlx5_core_mkey   mmkey;
-       struct ib_umem         *umem;
-       struct mlx5_shared_mr_info      *smr_info;
-       struct list_head        list;
-       struct mlx5_cache_ent  *cache_ent;
-       u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
-       struct mlx5_core_sig_ctx    *sig;
-       void                    *descs_alloc;
-       int                     access_flags; /* Needed for rereg MR */
-
-       struct mlx5_ib_mr      *parent;
-       /* Needed for IB_MR_TYPE_INTEGRITY */
-       struct mlx5_ib_mr      *pi_mr;
-       struct mlx5_ib_mr      *klm_mr;
-       struct mlx5_ib_mr      *mtt_mr;
-       u64                     data_iova;
-       u64                     pi_iova;
-
-       /* For ODP and implicit */
-       struct xarray           implicit_children;
-       union {
-               struct list_head elm;
-               struct work_struct work;
-       } odp_destroy;
-       struct ib_odp_counters  odp_stats;
-       bool                    is_odp_implicit;
+       struct ib_mr ibmr;
+       struct mlx5_core_mkey mmkey;
 
-       struct mlx5_async_work  cb_work;
+       /* User MR data */
+       struct mlx5_cache_ent *cache_ent;
+       struct ib_umem *umem;
+
+       /* This is zero'd when the MR is allocated */
+       struct {
+               /* Used only while the MR is in the cache */
+               struct {
+                       u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
+                       struct mlx5_async_work cb_work;
+                       /* Cache list element */
+                       struct list_head list;
+               };
+
+               /* Used only by kernel MRs (umem == NULL) */
+               struct {
+                       void *descs;
+                       void *descs_alloc;
+                       dma_addr_t desc_map;
+                       int max_descs;
+                       int ndescs;
+                       int desc_size;
+                       int access_mode;
+
+                       /* For Kernel IB_MR_TYPE_INTEGRITY */
+                       struct mlx5_core_sig_ctx *sig;
+                       struct mlx5_ib_mr *pi_mr;
+                       struct mlx5_ib_mr *klm_mr;
+                       struct mlx5_ib_mr *mtt_mr;
+                       u64 data_iova;
+                       u64 pi_iova;
+                       int meta_ndescs;
+                       int meta_length;
+                       int data_length;
+               };
+
+               /* Used only by User MRs (umem != NULL) */
+               struct {
+                       unsigned int page_shift;
+                       /* Current access_flags */
+                       int access_flags;
+
+                       /* For User ODP */
+                       struct mlx5_ib_mr *parent;
+                       struct xarray implicit_children;
+                       union {
+                               struct work_struct work;
+                       } odp_destroy;
+                       struct ib_odp_counters odp_stats;
+                       bool is_odp_implicit;
+               };
+       };
 };
 
+/* Zero the fields in the mr that are variant depending on usage */
+static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr)
+{
+       memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out));
+}
+
 static inline bool is_odp_mr(struct mlx5_ib_mr *mr)
 {
        return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index db05b0e0a8d7a19c56aa6a545e670d6d3e5b9e88..ea8f068a6da3e45f06ba22e207a06d279661ca1f 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -590,6 +590,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
                ent->available_mrs--;
                queue_adjust_cache_locked(ent);
                spin_unlock_irq(&ent->lock);
+
+               mlx5_clear_mr(mr);
        }
        mr->access_flags = access_flags;
        return mr;
@@ -615,16 +617,14 @@ static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
                        ent->available_mrs--;
                        queue_adjust_cache_locked(ent);
                        spin_unlock_irq(&ent->lock);
-                       break;
+                       mlx5_clear_mr(mr);
+                       return mr;
                }
                queue_adjust_cache_locked(ent);
                spin_unlock_irq(&ent->lock);
        }
-
-       if (!mr)
-               req_ent->miss++;
-
-       return mr;
+       req_ent->miss++;
+       return NULL;
 }
 
 static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
@@ -993,8 +993,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
 
        mr->ibmr.pd = pd;
        mr->umem = umem;
-       mr->access_flags = access_flags;
-       mr->desc_size = sizeof(struct mlx5_mtt);
        mr->mmkey.iova = iova;
        mr->mmkey.size = umem->length;
        mr->mmkey.pd = to_mpd(pd)->pdn;

diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index b103555b1f5d4cde00efb2e64c81be88379bd412..d98755e78362fb8c7058ced4b20f37447d70c671 100644 (file)
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -227,7 +227,6 @@ static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt)
 
        dma_fence_odp_mr(mr);
 
-       mr->parent = NULL;
        mlx5_mr_cache_free(mr_to_mdev(mr), mr);
        ib_umem_odp_release(odp);
 }