drivers/infiniband/hw/mlx5/dm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #include <rdma/uverbs_std_types.h>
   7 #include "dm.h"
   8
   9 #define UVERBS_MODULE_NAME mlx5_ib
  10 #include <rdma/uverbs_named_ioctl.h>
  11
  12 static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
  13                                 u64 length, u32 alignment)
  14 {
  15         struct mlx5_core_dev *dev = dm->dev;
  16         u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
  17                                         >> PAGE_SHIFT;
  18         u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
  19         u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
  20         u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
  21         u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
  22         u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
  23         u32 mlx5_alignment;
  24         u64 page_idx = 0;
  25         int ret = 0;
  26
  27         if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
  28                 return -EINVAL;
  29
  30         /* mlx5 device sets alignment as 64*2^driver_value
  31          * so normalizing is needed.
  32          */
  33         mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
  34                          alignment - MLX5_MEMIC_BASE_ALIGN;
  35         if (mlx5_alignment > max_alignment)
  36                 return -EINVAL;
  37
  38         MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
  39         MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
  40         MLX5_SET(alloc_memic_in, in, memic_size, length);
  41         MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
  42                  mlx5_alignment);
  43
  44         while (page_idx < num_memic_hw_pages) {
  45                 spin_lock(&dm->lock);
  46                 page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
  47                                                       num_memic_hw_pages,
  48                                                       page_idx,
  49                                                       num_pages, 0);
  50
  51                 if (page_idx < num_memic_hw_pages)
  52                         bitmap_set(dm->memic_alloc_pages,
  53                                    page_idx, num_pages);
  54
  55                 spin_unlock(&dm->lock);
  56
  57                 if (page_idx >= num_memic_hw_pages)
  58                         break;
  59
  60                 MLX5_SET64(alloc_memic_in, in, range_start_addr,
  61                            hw_start_addr + (page_idx * PAGE_SIZE));
  62
  63                 ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
  64                 if (ret) {
  65                         spin_lock(&dm->lock);
  66                         bitmap_clear(dm->memic_alloc_pages,
  67                                      page_idx, num_pages);
  68                         spin_unlock(&dm->lock);
  69
  70                         if (ret == -EAGAIN) {
  71                                 page_idx++;
  72                                 continue;
  73                         }
  74
  75                         return ret;
  76                 }
  77
  78                 *addr = dev->bar_addr +
  79                         MLX5_GET64(alloc_memic_out, out, memic_start_addr);
  80
  81                 return 0;
  82         }
  83
  84         return -ENOMEM;
  85 }
  86
  87 void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
  88                             u64 length)
  89 {
  90         struct mlx5_core_dev *dev = dm->dev;
  91         u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
  92         u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
  93         u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
  94         u64 start_page_idx;
  95         int err;
  96
  97         addr -= dev->bar_addr;
  98         start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
  99
 100         MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
 101         MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
 102         MLX5_SET(dealloc_memic_in, in, memic_size, length);
 103
 104         err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);
 105         if (err)
 106                 return;
 107
 108         spin_lock(&dm->lock);
 109         bitmap_clear(dm->memic_alloc_pages,
 110                      start_page_idx, num_pages);
 111         spin_unlock(&dm->lock);
 112 }
 113
 114 void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
 115                                u8 operation)
 116 {
 117         u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
 118         struct mlx5_core_dev *dev = dm->dev;
 119
 120         MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
 121         MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC);
 122         MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
 123         MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
 124
 125         mlx5_cmd_exec_in(dev, modify_memic, in);
 126 }
 127
 128 static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
 129                                    u8 operation, phys_addr_t *op_addr)
 130 {
 131         u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {};
 132         u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
 133         struct mlx5_core_dev *dev = dm->dev;
 134         int err;
 135
 136         MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
 137         MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC);
 138         MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
 139         MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
 140
 141         err = mlx5_cmd_exec_inout(dev, modify_memic, in, out);
 142         if (err)
 143                 return err;
 144
 145         *op_addr = dev->bar_addr +
 146                    MLX5_GET64(modify_memic_out, out, memic_operation_addr);
 147         return 0;
 148 }
 149
 150 static int add_dm_mmap_entry(struct ib_ucontext *context,
 151                              struct mlx5_user_mmap_entry *mentry, u8 mmap_flag,
 152                              size_t size, u64 address)
 153 {
 154         mentry->mmap_flag = mmap_flag;
 155         mentry->address = address;
 156
 157         return rdma_user_mmap_entry_insert_range(
 158                 context, &mentry->rdma_entry, size,
 159                 MLX5_IB_MMAP_DEVICE_MEM << 16,
 160                 (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
 161 }
 162
 163 static void mlx5_ib_dm_memic_free(struct kref *kref)
 164 {
 165         struct mlx5_ib_dm_memic *dm =
 166                 container_of(kref, struct mlx5_ib_dm_memic, ref);
 167         struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device);
 168
 169         mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size);
 170         kfree(dm);
 171 }
 172
 173 static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry,
 174                            struct uverbs_attr_bundle *attrs)
 175 {
 176         u64 start_offset;
 177         u16 page_idx;
 178         int err;
 179
 180         page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF;
 181         start_offset = op_entry->op_addr & ~PAGE_MASK;
 182         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
 183                              &page_idx, sizeof(page_idx));
 184         if (err)
 185                 return err;
 186
 187         return uverbs_copy_to(attrs,
 188                               MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
 189                               &start_offset, sizeof(start_offset));
 190 }
 191
 192 static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,
 193                            struct uverbs_attr_bundle *attrs)
 194 {
 195         struct mlx5_ib_dm_op_entry *op_entry;
 196
 197         op_entry = xa_load(&dm->ops, op);
 198         if (!op_entry)
 199                 return -ENOENT;
 200
 201         return copy_op_to_user(op_entry, attrs);
 202 }
 203
 204 static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(
 205         struct uverbs_attr_bundle *attrs)
 206 {
 207         struct ib_uobject *uobj = uverbs_attr_get_uobject(
 208                 attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE);
 209         struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
 210         struct ib_dm *ibdm = uobj->object;
 211         struct mlx5_ib_dm_memic *dm = to_memic(ibdm);
 212         struct mlx5_ib_dm_op_entry *op_entry;
 213         int err;
 214         u8 op;
 215
 216         err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP);
 217         if (err)
 218                 return err;
 219
 220         if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op)))
 221                 return -EOPNOTSUPP;
 222
 223         mutex_lock(&dm->ops_xa_lock);
 224         err = map_existing_op(dm, op, attrs);
 225         if (!err || err != -ENOENT)
 226                 goto err_unlock;
 227
 228         op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);
 229         if (!op_entry)
 230                 goto err_unlock;
 231
 232         err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op,
 233                                       &op_entry->op_addr);
 234         if (err) {
 235                 kfree(op_entry);
 236                 goto err_unlock;
 237         }
 238         op_entry->op = op;
 239         op_entry->dm = dm;
 240
 241         err = add_dm_mmap_entry(uobj->context, &op_entry->mentry,
 242                                 MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size,
 243                                 op_entry->op_addr & PAGE_MASK);
 244         if (err) {
 245                 mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op);
 246                 kfree(op_entry);
 247                 goto err_unlock;
 248         }
 249         /* From this point, entry will be freed by mmap_free */
 250         kref_get(&dm->ref);
 251
 252         err = copy_op_to_user(op_entry, attrs);
 253         if (err)
 254                 goto err_remove;
 255
 256         err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL);
 257         if (err)
 258                 goto err_remove;
 259         mutex_unlock(&dm->ops_xa_lock);
 260
 261         return 0;
 262
 263 err_remove:
 264         rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry);
 265 err_unlock:
 266         mutex_unlock(&dm->ops_xa_lock);
 267
 268         return err;
 269 }
 270
 271 static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx,
 272                                            struct ib_dm_alloc_attr *attr,
 273                                            struct uverbs_attr_bundle *attrs)
 274 {
 275         struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
 276         struct mlx5_ib_dm_memic *dm;
 277         u64 start_offset;
 278         u16 page_idx;
 279         int err;
 280         u64 address;
 281
 282         if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic))
 283                 return ERR_PTR(-EOPNOTSUPP);
 284
 285         dm = kzalloc(sizeof(*dm), GFP_KERNEL);
 286         if (!dm)
 287                 return ERR_PTR(-ENOMEM);
 288
 289         dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;
 290         dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
 291         dm->base.ibdm.device = ctx->device;
 292
 293         kref_init(&dm->ref);
 294         xa_init(&dm->ops);
 295         mutex_init(&dm->ops_xa_lock);
 296         dm->req_length = attr->length;
 297
 298         err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr,
 299                                    dm->base.size, attr->alignment);
 300         if (err) {
 301                 kfree(dm);
 302                 return ERR_PTR(err);
 303         }
 304
 305         address = dm->base.dev_addr & PAGE_MASK;
 306         err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC,
 307                                 dm->base.size, address);
 308         if (err) {
 309                 mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size);
 310                 kfree(dm);
 311                 return ERR_PTR(err);
 312         }
 313
 314         page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
 315         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
 316                              &page_idx, sizeof(page_idx));
 317         if (err)
 318                 goto err_copy;
 319
 320         start_offset = dm->base.dev_addr & ~PAGE_MASK;
 321         err = uverbs_copy_to(attrs,
 322                              MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 323                              &start_offset, sizeof(start_offset));
 324         if (err)
 325                 goto err_copy;
 326
 327         return &dm->base.ibdm;
 328
 329 err_copy:
 330         rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
 331         return ERR_PTR(err);
 332 }
 333
 334 static enum mlx5_sw_icm_type get_icm_type(int uapi_type)
 335 {
 336         return uapi_type == MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM ?
 337                        MLX5_SW_ICM_TYPE_STEERING :
 338                        MLX5_SW_ICM_TYPE_HEADER_MODIFY;
 339 }
 340
 341 static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
 342                                             struct ib_dm_alloc_attr *attr,
 343                                             struct uverbs_attr_bundle *attrs,
 344                                             int type)
 345 {
 346         struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
 347         enum mlx5_sw_icm_type icm_type = get_icm_type(type);
 348         struct mlx5_ib_dm_icm *dm;
 349         u64 act_size;
 350         int err;
 351
 352         dm = kzalloc(sizeof(*dm), GFP_KERNEL);
 353         if (!dm)
 354                 return ERR_PTR(-ENOMEM);
 355
 356         dm->base.type = type;
 357         dm->base.ibdm.device = ctx->device;
 358
 359         if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW)) {
 360                 err = -EPERM;
 361                 goto free;
 362         }
 363
 364         if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) ||
 365               MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) ||
 366               MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) ||
 367               MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) {
 368                 err = -EOPNOTSUPP;
 369                 goto free;
 370         }
 371
 372         /* Allocation size must a multiple of the basic block size
 373          * and a power of 2.
 374          */
 375         act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
 376         act_size = roundup_pow_of_two(act_size);
 377
 378         dm->base.size = act_size;
 379         err = mlx5_dm_sw_icm_alloc(dev, icm_type, act_size, attr->alignment,
 380                                    to_mucontext(ctx)->devx_uid,
 381                                    &dm->base.dev_addr, &dm->obj_id);
 382         if (err)
 383                 goto free;
 384
 385         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 386                              &dm->base.dev_addr, sizeof(dm->base.dev_addr));
 387         if (err) {
 388                 mlx5_dm_sw_icm_dealloc(dev, icm_type, dm->base.size,
 389                                        to_mucontext(ctx)->devx_uid,
 390                                        dm->base.dev_addr, dm->obj_id);
 391                 goto free;
 392         }
 393         return &dm->base.ibdm;
 394 free:
 395         kfree(dm);
 396         return ERR_PTR(err);
 397 }
 398
 399 struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
 400                                struct ib_ucontext *context,
 401                                struct ib_dm_alloc_attr *attr,
 402                                struct uverbs_attr_bundle *attrs)
 403 {
 404         enum mlx5_ib_uapi_dm_type type;
 405         int err;
 406
 407         err = uverbs_get_const_default(&type, attrs,
 408                                        MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
 409                                        MLX5_IB_UAPI_DM_TYPE_MEMIC);
 410         if (err)
 411                 return ERR_PTR(err);
 412
 413         mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
 414                     type, attr->length, attr->alignment);
 415
 416         switch (type) {
 417         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
 418                 return handle_alloc_dm_memic(context, attr, attrs);
 419         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
 420                 return handle_alloc_dm_sw_icm(context, attr, attrs, type);
 421         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
 422                 return handle_alloc_dm_sw_icm(context, attr, attrs, type);
 423         default:
 424                 return ERR_PTR(-EOPNOTSUPP);
 425         }
 426 }
 427
 428 static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)
 429 {
 430         struct mlx5_ib_dm_op_entry *entry;
 431         unsigned long idx;
 432
 433         mutex_lock(&dm->ops_xa_lock);
 434         xa_for_each(&dm->ops, idx, entry) {
 435                 xa_erase(&dm->ops, idx);
 436                 rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry);
 437         }
 438         mutex_unlock(&dm->ops_xa_lock);
 439 }
 440
 441 static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)
 442 {
 443         dm_memic_remove_ops(dm);
 444         rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
 445 }
 446
 447 static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx,
 448                                struct mlx5_ib_dm_icm *dm)
 449 {
 450         enum mlx5_sw_icm_type type = get_icm_type(dm->base.type);
 451         struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev;
 452         int err;
 453
 454         err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid,
 455                                      dm->base.dev_addr, dm->obj_id);
 456         if (!err)
 457                 kfree(dm);
 458         return 0;
 459 }
 460
 461 static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm,
 462                               struct uverbs_attr_bundle *attrs)
 463 {
 464         struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
 465                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
 466         struct mlx5_ib_dm *dm = to_mdm(ibdm);
 467
 468         switch (dm->type) {
 469         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
 470                 mlx5_dm_memic_dealloc(to_memic(ibdm));
 471                 return 0;
 472         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
 473         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
 474                 return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm));
 475         default:
 476                 return -EOPNOTSUPP;
 477         }
 478 }
 479
 480 static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)(
 481         struct uverbs_attr_bundle *attrs)
 482 {
 483         struct ib_dm *ibdm =
 484                 uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE);
 485         struct mlx5_ib_dm *dm = to_mdm(ibdm);
 486         struct mlx5_ib_dm_memic *memic;
 487         u64 start_offset;
 488         u16 page_idx;
 489         int err;
 490
 491         if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC)
 492                 return -EOPNOTSUPP;
 493
 494         memic = to_memic(ibdm);
 495         page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF;
 496         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
 497                              &page_idx, sizeof(page_idx));
 498         if (err)
 499                 return err;
 500
 501         start_offset = memic->base.dev_addr & ~PAGE_MASK;
 502         err =  uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
 503                               &start_offset, sizeof(start_offset));
 504         if (err)
 505                 return err;
 506
 507         return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
 508                               &memic->req_length,
 509                               sizeof(memic->req_length));
 510 }
 511
 512 void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
 513                           struct mlx5_user_mmap_entry *mentry)
 514 {
 515         struct mlx5_ib_dm_op_entry *op_entry;
 516         struct mlx5_ib_dm_memic *mdm;
 517
 518         switch (mentry->mmap_flag) {
 519         case MLX5_IB_MMAP_TYPE_MEMIC:
 520                 mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry);
 521                 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
 522                 break;
 523         case MLX5_IB_MMAP_TYPE_MEMIC_OP:
 524                 op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry,
 525                                         mentry);
 526                 mdm = op_entry->dm;
 527                 mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr,
 528                                           op_entry->op);
 529                 kfree(op_entry);
 530                 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
 531                 break;
 532         default:
 533                 WARN_ON(true);
 534         }
 535 }
 536
 537 DECLARE_UVERBS_NAMED_METHOD(
 538         MLX5_IB_METHOD_DM_QUERY,
 539         UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM,
 540                         UVERBS_ACCESS_READ, UA_MANDATORY),
 541         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
 542                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
 543         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
 544                             UVERBS_ATTR_TYPE(u16), UA_MANDATORY),
 545         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
 546                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY));
 547
 548 ADD_UVERBS_ATTRIBUTES_SIMPLE(
 549         mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,
 550         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 551                             UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
 552         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
 553                             UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
 554         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
 555                              enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));
 556
 557 DECLARE_UVERBS_NAMED_METHOD(
 558         MLX5_IB_METHOD_DM_MAP_OP_ADDR,
 559         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE,
 560                         UVERBS_OBJECT_DM,
 561                         UVERBS_ACCESS_READ,
 562                         UA_MANDATORY),
 563         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
 564                            UVERBS_ATTR_TYPE(u8),
 565                            UA_MANDATORY),
 566         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
 567                             UVERBS_ATTR_TYPE(u64),
 568                             UA_MANDATORY),
 569         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
 570                             UVERBS_ATTR_TYPE(u16),
 571                             UA_OPTIONAL));
 572
 573 DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM,
 574                               &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR),
 575                               &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY));
 576
 577 const struct uapi_definition mlx5_ib_dm_defs[] = {
 578         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
 579         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM),
 580         {},
 581 };
 582
 583 const struct ib_device_ops mlx5_ib_dev_dm_ops = {
 584         .alloc_dm = mlx5_ib_alloc_dm,
 585         .dealloc_dm = mlx5_ib_dealloc_dm,
 586         .reg_dm_mr = mlx5_ib_reg_dm_mr,
 587 };