drivers/infiniband/hw/mlx5/dm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #include <rdma/uverbs_std_types.h>
   7 #include "dm.h"
   8
   9 #define UVERBS_MODULE_NAME mlx5_ib
  10 #include <rdma/uverbs_named_ioctl.h>
  11
  12 static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
  13                                 u64 length, u32 alignment)
  14 {
  15         struct mlx5_core_dev *dev = dm->dev;
  16         u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
  17                                         >> PAGE_SHIFT;
  18         u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
  19         u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
  20         u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
  21         u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
  22         u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
  23         u32 mlx5_alignment;
  24         u64 page_idx = 0;
  25         int ret = 0;
  26
  27         if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
  28                 return -EINVAL;
  29
  30         /* mlx5 device sets alignment as 64*2^driver_value
  31          * so normalizing is needed.
  32          */
  33         mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
  34                          alignment - MLX5_MEMIC_BASE_ALIGN;
  35         if (mlx5_alignment > max_alignment)
  36                 return -EINVAL;
  37
  38         MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
  39         MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
  40         MLX5_SET(alloc_memic_in, in, memic_size, length);
  41         MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
  42                  mlx5_alignment);
  43
  44         while (page_idx < num_memic_hw_pages) {
  45                 spin_lock(&dm->lock);
  46                 page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
  47                                                       num_memic_hw_pages,
  48                                                       page_idx,
  49                                                       num_pages, 0);
  50
  51                 if (page_idx < num_memic_hw_pages)
  52                         bitmap_set(dm->memic_alloc_pages,
  53                                    page_idx, num_pages);
  54
  55                 spin_unlock(&dm->lock);
  56
  57                 if (page_idx >= num_memic_hw_pages)
  58                         break;
  59
  60                 MLX5_SET64(alloc_memic_in, in, range_start_addr,
  61                            hw_start_addr + (page_idx * PAGE_SIZE));
  62
  63                 ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
  64                 if (ret) {
  65                         spin_lock(&dm->lock);
  66                         bitmap_clear(dm->memic_alloc_pages,
  67                                      page_idx, num_pages);
  68                         spin_unlock(&dm->lock);
  69
  70                         if (ret == -EAGAIN) {
  71                                 page_idx++;
  72                                 continue;
  73                         }
  74
  75                         return ret;
  76                 }
  77
  78                 *addr = dev->bar_addr +
  79                         MLX5_GET64(alloc_memic_out, out, memic_start_addr);
  80
  81                 return 0;
  82         }
  83
  84         return -ENOMEM;
  85 }
  86
  87 void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
  88                             u64 length)
  89 {
  90         struct mlx5_core_dev *dev = dm->dev;
  91         u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
  92         u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
  93         u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
  94         u64 start_page_idx;
  95         int err;
  96
  97         addr -= dev->bar_addr;
  98         start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
  99
 100         MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
 101         MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
 102         MLX5_SET(dealloc_memic_in, in, memic_size, length);
 103
 104         err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);
 105         if (err)
 106                 return;
 107
 108         spin_lock(&dm->lock);
 109         bitmap_clear(dm->memic_alloc_pages,
 110                      start_page_idx, num_pages);
 111         spin_unlock(&dm->lock);
 112 }
 113
 114 void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
 115                                u8 operation)
 116 {
 117         u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
 118         struct mlx5_core_dev *dev = dm->dev;
 119
 120         MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
 121         MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC);
 122         MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
 123         MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
 124
 125         mlx5_cmd_exec_in(dev, modify_memic, in);
 126 }
 127
 128 static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
 129                                    u8 operation, phys_addr_t *op_addr)
 130 {
 131         u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {};
 132         u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
 133         struct mlx5_core_dev *dev = dm->dev;
 134         int err;
 135
 136         MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
 137         MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC);
 138         MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
 139         MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
 140
 141         err = mlx5_cmd_exec_inout(dev, modify_memic, in, out);
 142         if (err)
 143                 return err;
 144
 145         *op_addr = dev->bar_addr +
 146                    MLX5_GET64(modify_memic_out, out, memic_operation_addr);
 147         return 0;
 148 }
 149
 150 static int add_dm_mmap_entry(struct ib_ucontext *context,
 151                              struct mlx5_user_mmap_entry *mentry, u8 mmap_flag,
 152                              size_t size, u64 address)
 153 {
 154         mentry->mmap_flag = mmap_flag;
 155         mentry->address = address;
 156
 157         return rdma_user_mmap_entry_insert_range(
 158                 context, &mentry->rdma_entry, size,
 159                 MLX5_IB_MMAP_DEVICE_MEM << 16,
 160                 (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
 161 }
 162
 163 static void mlx5_ib_dm_memic_free(struct kref *kref)
 164 {
 165         struct mlx5_ib_dm_memic *dm =
 166                 container_of(kref, struct mlx5_ib_dm_memic, ref);
 167         struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device);
 168
 169         mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size);
 170         kfree(dm);
 171 }
 172
 173 static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry,
 174                            struct uverbs_attr_bundle *attrs)
 175 {
 176         u64 start_offset;
 177         u16 page_idx;
 178         int err;
 179
 180         page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF;
 181         start_offset = op_entry->op_addr & ~PAGE_MASK;
 182         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
 183                              &page_idx, sizeof(page_idx));
 184         if (err)
 185                 return err;
 186
 187         return uverbs_copy_to(attrs,
 188                               MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
 189                               &start_offset, sizeof(start_offset));
 190 }
 191
 192 static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,
 193                            struct uverbs_attr_bundle *attrs)
 194 {
 195         struct mlx5_ib_dm_op_entry *op_entry;
 196
 197         op_entry = xa_load(&dm->ops, op);
 198         if (!op_entry)
 199                 return -ENOENT;
 200
 201         return copy_op_to_user(op_entry, attrs);
 202 }
 203
 204 static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(
 205         struct uverbs_attr_bundle *attrs)
 206 {
 207         struct ib_uobject *uobj = uverbs_attr_get_uobject(
 208                 attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE);
 209         struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
 210         struct ib_dm *ibdm = uobj->object;
 211         struct mlx5_ib_dm_memic *dm = to_memic(ibdm);
 212         struct mlx5_ib_dm_op_entry *op_entry;
 213         int err;
 214         u8 op;
 215
 216         err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP);
 217         if (err)
 218                 return err;
 219
 220         if (op >= BITS_PER_TYPE(u32))
 221                 return -EOPNOTSUPP;
 222
 223         if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op)))
 224                 return -EOPNOTSUPP;
 225
 226         mutex_lock(&dm->ops_xa_lock);
 227         err = map_existing_op(dm, op, attrs);
 228         if (!err || err != -ENOENT)
 229                 goto err_unlock;
 230
 231         op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);
 232         if (!op_entry)
 233                 goto err_unlock;
 234
 235         err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op,
 236                                       &op_entry->op_addr);
 237         if (err) {
 238                 kfree(op_entry);
 239                 goto err_unlock;
 240         }
 241         op_entry->op = op;
 242         op_entry->dm = dm;
 243
 244         err = add_dm_mmap_entry(uobj->context, &op_entry->mentry,
 245                                 MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size,
 246                                 op_entry->op_addr & PAGE_MASK);
 247         if (err) {
 248                 mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op);
 249                 kfree(op_entry);
 250                 goto err_unlock;
 251         }
 252         /* From this point, entry will be freed by mmap_free */
 253         kref_get(&dm->ref);
 254
 255         err = copy_op_to_user(op_entry, attrs);
 256         if (err)
 257                 goto err_remove;
 258
 259         err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL);
 260         if (err)
 261                 goto err_remove;
 262         mutex_unlock(&dm->ops_xa_lock);
 263
 264         return 0;
 265
 266 err_remove:
 267         rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry);
 268 err_unlock:
 269         mutex_unlock(&dm->ops_xa_lock);
 270
 271         return err;
 272 }
 273
 274 static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx,
 275                                            struct ib_dm_alloc_attr *attr,
 276                                            struct uverbs_attr_bundle *attrs)
 277 {
 278         struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
 279         struct mlx5_ib_dm_memic *dm;
 280         u64 start_offset;
 281         u16 page_idx;
 282         int err;
 283         u64 address;
 284
 285         if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic))
 286                 return ERR_PTR(-EOPNOTSUPP);
 287
 288         dm = kzalloc(sizeof(*dm), GFP_KERNEL);
 289         if (!dm)
 290                 return ERR_PTR(-ENOMEM);
 291
 292         dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;
 293         dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
 294         dm->base.ibdm.device = ctx->device;
 295
 296         kref_init(&dm->ref);
 297         xa_init(&dm->ops);
 298         mutex_init(&dm->ops_xa_lock);
 299         dm->req_length = attr->length;
 300
 301         err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr,
 302                                    dm->base.size, attr->alignment);
 303         if (err) {
 304                 kfree(dm);
 305                 return ERR_PTR(err);
 306         }
 307
 308         address = dm->base.dev_addr & PAGE_MASK;
 309         err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC,
 310                                 dm->base.size, address);
 311         if (err) {
 312                 mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size);
 313                 kfree(dm);
 314                 return ERR_PTR(err);
 315         }
 316
 317         page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
 318         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
 319                              &page_idx, sizeof(page_idx));
 320         if (err)
 321                 goto err_copy;
 322
 323         start_offset = dm->base.dev_addr & ~PAGE_MASK;
 324         err = uverbs_copy_to(attrs,
 325                              MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 326                              &start_offset, sizeof(start_offset));
 327         if (err)
 328                 goto err_copy;
 329
 330         return &dm->base.ibdm;
 331
 332 err_copy:
 333         rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
 334         return ERR_PTR(err);
 335 }
 336
 337 static enum mlx5_sw_icm_type get_icm_type(int uapi_type)
 338 {
 339         return uapi_type == MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM ?
 340                        MLX5_SW_ICM_TYPE_STEERING :
 341                        MLX5_SW_ICM_TYPE_HEADER_MODIFY;
 342 }
 343
 344 static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
 345                                             struct ib_dm_alloc_attr *attr,
 346                                             struct uverbs_attr_bundle *attrs,
 347                                             int type)
 348 {
 349         struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
 350         enum mlx5_sw_icm_type icm_type = get_icm_type(type);
 351         struct mlx5_ib_dm_icm *dm;
 352         u64 act_size;
 353         int err;
 354
 355         dm = kzalloc(sizeof(*dm), GFP_KERNEL);
 356         if (!dm)
 357                 return ERR_PTR(-ENOMEM);
 358
 359         dm->base.type = type;
 360         dm->base.ibdm.device = ctx->device;
 361
 362         if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW)) {
 363                 err = -EPERM;
 364                 goto free;
 365         }
 366
 367         if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) ||
 368               MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) ||
 369               MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) ||
 370               MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) {
 371                 err = -EOPNOTSUPP;
 372                 goto free;
 373         }
 374
 375         /* Allocation size must a multiple of the basic block size
 376          * and a power of 2.
 377          */
 378         act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
 379         act_size = roundup_pow_of_two(act_size);
 380
 381         dm->base.size = act_size;
 382         err = mlx5_dm_sw_icm_alloc(dev, icm_type, act_size, attr->alignment,
 383                                    to_mucontext(ctx)->devx_uid,
 384                                    &dm->base.dev_addr, &dm->obj_id);
 385         if (err)
 386                 goto free;
 387
 388         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 389                              &dm->base.dev_addr, sizeof(dm->base.dev_addr));
 390         if (err) {
 391                 mlx5_dm_sw_icm_dealloc(dev, icm_type, dm->base.size,
 392                                        to_mucontext(ctx)->devx_uid,
 393                                        dm->base.dev_addr, dm->obj_id);
 394                 goto free;
 395         }
 396         return &dm->base.ibdm;
 397 free:
 398         kfree(dm);
 399         return ERR_PTR(err);
 400 }
 401
 402 struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
 403                                struct ib_ucontext *context,
 404                                struct ib_dm_alloc_attr *attr,
 405                                struct uverbs_attr_bundle *attrs)
 406 {
 407         enum mlx5_ib_uapi_dm_type type;
 408         int err;
 409
 410         err = uverbs_get_const_default(&type, attrs,
 411                                        MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
 412                                        MLX5_IB_UAPI_DM_TYPE_MEMIC);
 413         if (err)
 414                 return ERR_PTR(err);
 415
 416         mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
 417                     type, attr->length, attr->alignment);
 418
 419         switch (type) {
 420         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
 421                 return handle_alloc_dm_memic(context, attr, attrs);
 422         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
 423                 return handle_alloc_dm_sw_icm(context, attr, attrs, type);
 424         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
 425                 return handle_alloc_dm_sw_icm(context, attr, attrs, type);
 426         default:
 427                 return ERR_PTR(-EOPNOTSUPP);
 428         }
 429 }
 430
 431 static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)
 432 {
 433         struct mlx5_ib_dm_op_entry *entry;
 434         unsigned long idx;
 435
 436         mutex_lock(&dm->ops_xa_lock);
 437         xa_for_each(&dm->ops, idx, entry) {
 438                 xa_erase(&dm->ops, idx);
 439                 rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry);
 440         }
 441         mutex_unlock(&dm->ops_xa_lock);
 442 }
 443
 444 static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)
 445 {
 446         dm_memic_remove_ops(dm);
 447         rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
 448 }
 449
 450 static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx,
 451                                struct mlx5_ib_dm_icm *dm)
 452 {
 453         enum mlx5_sw_icm_type type = get_icm_type(dm->base.type);
 454         struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev;
 455         int err;
 456
 457         err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid,
 458                                      dm->base.dev_addr, dm->obj_id);
 459         if (!err)
 460                 kfree(dm);
 461         return 0;
 462 }
 463
 464 static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm,
 465                               struct uverbs_attr_bundle *attrs)
 466 {
 467         struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
 468                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
 469         struct mlx5_ib_dm *dm = to_mdm(ibdm);
 470
 471         switch (dm->type) {
 472         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
 473                 mlx5_dm_memic_dealloc(to_memic(ibdm));
 474                 return 0;
 475         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
 476         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
 477                 return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm));
 478         default:
 479                 return -EOPNOTSUPP;
 480         }
 481 }
 482
 483 static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)(
 484         struct uverbs_attr_bundle *attrs)
 485 {
 486         struct ib_dm *ibdm =
 487                 uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE);
 488         struct mlx5_ib_dm *dm = to_mdm(ibdm);
 489         struct mlx5_ib_dm_memic *memic;
 490         u64 start_offset;
 491         u16 page_idx;
 492         int err;
 493
 494         if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC)
 495                 return -EOPNOTSUPP;
 496
 497         memic = to_memic(ibdm);
 498         page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF;
 499         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
 500                              &page_idx, sizeof(page_idx));
 501         if (err)
 502                 return err;
 503
 504         start_offset = memic->base.dev_addr & ~PAGE_MASK;
 505         err =  uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
 506                               &start_offset, sizeof(start_offset));
 507         if (err)
 508                 return err;
 509
 510         return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
 511                               &memic->req_length,
 512                               sizeof(memic->req_length));
 513 }
 514
 515 void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
 516                           struct mlx5_user_mmap_entry *mentry)
 517 {
 518         struct mlx5_ib_dm_op_entry *op_entry;
 519         struct mlx5_ib_dm_memic *mdm;
 520
 521         switch (mentry->mmap_flag) {
 522         case MLX5_IB_MMAP_TYPE_MEMIC:
 523                 mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry);
 524                 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
 525                 break;
 526         case MLX5_IB_MMAP_TYPE_MEMIC_OP:
 527                 op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry,
 528                                         mentry);
 529                 mdm = op_entry->dm;
 530                 mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr,
 531                                           op_entry->op);
 532                 kfree(op_entry);
 533                 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
 534                 break;
 535         default:
 536                 WARN_ON(true);
 537         }
 538 }
 539
 540 DECLARE_UVERBS_NAMED_METHOD(
 541         MLX5_IB_METHOD_DM_QUERY,
 542         UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM,
 543                         UVERBS_ACCESS_READ, UA_MANDATORY),
 544         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
 545                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
 546         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
 547                             UVERBS_ATTR_TYPE(u16), UA_MANDATORY),
 548         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
 549                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY));
 550
 551 ADD_UVERBS_ATTRIBUTES_SIMPLE(
 552         mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,
 553         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 554                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
 555         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
 556                             UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
 557         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
 558                              enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));
 559
 560 DECLARE_UVERBS_NAMED_METHOD(
 561         MLX5_IB_METHOD_DM_MAP_OP_ADDR,
 562         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE,
 563                         UVERBS_OBJECT_DM,
 564                         UVERBS_ACCESS_READ,
 565                         UA_MANDATORY),
 566         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
 567                            UVERBS_ATTR_TYPE(u8),
 568                            UA_MANDATORY),
 569         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
 570                             UVERBS_ATTR_TYPE(u64),
 571                             UA_MANDATORY),
 572         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
 573                             UVERBS_ATTR_TYPE(u16),
 574                             UA_OPTIONAL));
 575
 576 DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM,
 577                               &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR),
 578                               &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY));
 579
 580 const struct uapi_definition mlx5_ib_dm_defs[] = {
 581         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
 582         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM),
 583         {},
 584 };
 585
 586 const struct ib_device_ops mlx5_ib_dev_dm_ops = {
 587         .alloc_dm = mlx5_ib_alloc_dm,
 588         .dealloc_dm = mlx5_ib_dealloc_dm,
 589         .reg_dm_mr = mlx5_ib_reg_dm_mr,
 590 };