RDMA/mlx5: Zero out ODP related items in the mlx5_ib_mr

All of the ODP code assumes when it calls mlx5_mr_cache_alloc() the ODP
related fields are zero'd. This is true if the MR was just allocated, but
if the MR is recycled through the cache then the values are never zero'd.

This causes a bug in the odp_stats, they don't reset when the MR is
reallocated, also is_odp_implicit is never 0'd.

So we can use memset on a block of the mlx5_ib_mr reorganize the structure
to put all the data that can be zero'd by the cache at the end.

It is organized as an anonymous struct because the next patch will make
this a union.

Delete the unused smr_info. Don't set the kernel only desc_size on the
user path. No longer any need to zero mr->parent before freeing it, the
memset() will get it now.

Fixes: a3de94e3d61e ("IB/mlx5: Introduce ODP diagnostic counters")
Link: https://lore.kernel.org/r/20210304120745.1090751-2-leon@kernel.org
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 88cc26e..b085c02 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -547,11 +547,6 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
        return container_of(wr, struct mlx5_umr_wr, wr);
 }
 
-struct mlx5_shared_mr_info {
-       int mr_id;
-       struct ib_umem          *umem;
-};
-
 enum mlx5_ib_cq_pr_flags {
        MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0,
 };
@@ -654,47 +649,69 @@ struct mlx5_ib_dm {
        atomic64_add(value, &((mr)->odp_stats.counter_name))
 
 struct mlx5_ib_mr {
-       struct ib_mr            ibmr;
-       void                    *descs;
-       dma_addr_t              desc_map;
-       int                     ndescs;
-       int                     data_length;
-       int                     meta_ndescs;
-       int                     meta_length;
-       int                     max_descs;
-       int                     desc_size;
-       int                     access_mode;
-       unsigned int            page_shift;
-       struct mlx5_core_mkey   mmkey;
-       struct ib_umem         *umem;
-       struct mlx5_shared_mr_info      *smr_info;
-       struct list_head        list;
-       struct mlx5_cache_ent  *cache_ent;
-       u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
-       struct mlx5_core_sig_ctx    *sig;
-       void                    *descs_alloc;
-       int                     access_flags; /* Needed for rereg MR */
-
-       struct mlx5_ib_mr      *parent;
-       /* Needed for IB_MR_TYPE_INTEGRITY */
-       struct mlx5_ib_mr      *pi_mr;
-       struct mlx5_ib_mr      *klm_mr;
-       struct mlx5_ib_mr      *mtt_mr;
-       u64                     data_iova;
-       u64                     pi_iova;
-
-       /* For ODP and implicit */
-       struct xarray           implicit_children;
-       union {
-               struct list_head elm;
-               struct work_struct work;
-       } odp_destroy;
-       struct ib_odp_counters  odp_stats;
-       bool                    is_odp_implicit;
+       struct ib_mr ibmr;
+       struct mlx5_core_mkey mmkey;
 
-       struct mlx5_async_work  cb_work;
+       /* User MR data */
+       struct mlx5_cache_ent *cache_ent;
+       struct ib_umem *umem;
+
+       /* This is zero'd when the MR is allocated */
+       struct {
+               /* Used only while the MR is in the cache */
+               struct {
+                       u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
+                       struct mlx5_async_work cb_work;
+                       /* Cache list element */
+                       struct list_head list;
+               };
+
+               /* Used only by kernel MRs (umem == NULL) */
+               struct {
+                       void *descs;
+                       void *descs_alloc;
+                       dma_addr_t desc_map;
+                       int max_descs;
+                       int ndescs;
+                       int desc_size;
+                       int access_mode;
+
+                       /* For Kernel IB_MR_TYPE_INTEGRITY */
+                       struct mlx5_core_sig_ctx *sig;
+                       struct mlx5_ib_mr *pi_mr;
+                       struct mlx5_ib_mr *klm_mr;
+                       struct mlx5_ib_mr *mtt_mr;
+                       u64 data_iova;
+                       u64 pi_iova;
+                       int meta_ndescs;
+                       int meta_length;
+                       int data_length;
+               };
+
+               /* Used only by User MRs (umem != NULL) */
+               struct {
+                       unsigned int page_shift;
+                       /* Current access_flags */
+                       int access_flags;
+
+                       /* For User ODP */
+                       struct mlx5_ib_mr *parent;
+                       struct xarray implicit_children;
+                       union {
+                               struct work_struct work;
+                       } odp_destroy;
+                       struct ib_odp_counters odp_stats;
+                       bool is_odp_implicit;
+               };
+       };
 };
 
+/* Zero the fields in the mr that are variant depending on usage */
+static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr)
+{
+       memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out));
+}
+
 static inline bool is_odp_mr(struct mlx5_ib_mr *mr)
 {
        return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index db05b0e..ea8f068 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -590,6 +590,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
                ent->available_mrs--;
                queue_adjust_cache_locked(ent);
                spin_unlock_irq(&ent->lock);
+
+               mlx5_clear_mr(mr);
        }
        mr->access_flags = access_flags;
        return mr;
@@ -615,16 +617,14 @@ static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
                        ent->available_mrs--;
                        queue_adjust_cache_locked(ent);
                        spin_unlock_irq(&ent->lock);
-                       break;
+                       mlx5_clear_mr(mr);
+                       return mr;
                }
                queue_adjust_cache_locked(ent);
                spin_unlock_irq(&ent->lock);
        }
-
-       if (!mr)
-               req_ent->miss++;
-
-       return mr;
+       req_ent->miss++;
+       return NULL;
 }
 
 static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
@@ -993,8 +993,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
 
        mr->ibmr.pd = pd;
        mr->umem = umem;
-       mr->access_flags = access_flags;
-       mr->desc_size = sizeof(struct mlx5_mtt);
        mr->mmkey.iova = iova;
        mr->mmkey.size = umem->length;
        mr->mmkey.pd = to_mpd(pd)->pdn;

diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index b103555..d98755e 100644 (file)
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -227,7 +227,6 @@ static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt)
 
        dma_fence_odp_mr(mr);
 
-       mr->parent = NULL;
        mlx5_mr_cache_free(mr_to_mdev(mr), mr);
        ib_umem_odp_release(odp);
 }