drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/mlx5_user_ioctl_verbs.h>
  12 #include <rdma/ib_umem.h>
  13 #include <rdma/uverbs_std_types.h>
  14 #include <linux/mlx5/driver.h>
  15 #include <linux/mlx5/fs.h>
  16 #include "mlx5_ib.h"
  17
  18 #define UVERBS_MODULE_NAME mlx5_ib
  19 #include <rdma/uverbs_named_ioctl.h>
  20
  21 enum devx_obj_flags {
  22         DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  23         DEVX_OBJ_FLAGS_DCT = 1 << 1,
  24 };
  25
  26 struct devx_async_data {
  27         struct mlx5_ib_dev *mdev;
  28         struct list_head list;
  29         struct ib_uobject *fd_uobj;
  30         struct mlx5_async_work cb_work;
  31         u16 cmd_out_len;
  32         /* must be last field in this structure */
  33         struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  34 };
  35
  36 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
  37 struct devx_obj {
  38         struct mlx5_core_dev    *mdev;
  39         u64                     obj_id;
  40         u32                     dinlen; /* destroy inbox length */
  41         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
  42         u32                     flags;
  43         union {
  44                 struct mlx5_ib_devx_mr  devx_mr;
  45                 struct mlx5_core_dct    core_dct;
  46         };
  47 };
  48
  49 struct devx_umem {
  50         struct mlx5_core_dev            *mdev;
  51         struct ib_umem                  *umem;
  52         u32                             page_offset;
  53         int                             page_shift;
  54         int                             ncont;
  55         u32                             dinlen;
  56         u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
  57 };
  58
  59 struct devx_umem_reg_cmd {
  60         void                            *in;
  61         u32                             inlen;
  62         u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
  63 };
  64
  65 static struct mlx5_ib_ucontext *
  66 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
  67 {
  68         return to_mucontext(ib_uverbs_get_ucontext(attrs));
  69 }
  70
  71 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
  72 {
  73         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
  74         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
  75         void *uctx;
  76         int err;
  77         u16 uid;
  78         u32 cap = 0;
  79
  80         /* 0 means not supported */
  81         if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
  82                 return -EINVAL;
  83
  84         uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
  85         if (is_user && capable(CAP_NET_RAW) &&
  86             (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
  87                 cap |= MLX5_UCTX_CAP_RAW_TX;
  88         if (is_user && capable(CAP_SYS_RAWIO) &&
  89             (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
  90              MLX5_UCTX_CAP_INTERNAL_DEV_RES))
  91                 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
  92
  93         MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
  94         MLX5_SET(uctx, uctx, cap, cap);
  95
  96         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
  97         if (err)
  98                 return err;
  99
 100         uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 101         return uid;
 102 }
 103
 104 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 105 {
 106         u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
 107         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 108
 109         MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 110         MLX5_SET(destroy_uctx_in, in, uid, uid);
 111
 112         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 113 }
 114
 115 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
 116 {
 117         struct devx_obj *devx_obj = obj;
 118         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 119
 120         switch (opcode) {
 121         case MLX5_CMD_OP_DESTROY_TIR:
 122                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 123                 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
 124                                     obj_id);
 125                 return true;
 126
 127         case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 128                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 129                 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
 130                                     table_id);
 131                 return true;
 132         default:
 133                 return false;
 134         }
 135 }
 136
 137 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id)
 138 {
 139         struct devx_obj *devx_obj = obj;
 140         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 141
 142         if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
 143                 *counter_id = MLX5_GET(dealloc_flow_counter_in,
 144                                        devx_obj->dinbox,
 145                                        flow_counter_id);
 146                 return true;
 147         }
 148
 149         return false;
 150 }
 151
 152 /*
 153  * As the obj_id in the firmware is not globally unique the object type
 154  * must be considered upon checking for a valid object id.
 155  * For that the opcode of the creator command is encoded as part of the obj_id.
 156  */
 157 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
 158 {
 159         return ((u64)opcode << 32) | obj_id;
 160 }
 161
 162 static u64 devx_get_obj_id(const void *in)
 163 {
 164         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 165         u64 obj_id;
 166
 167         switch (opcode) {
 168         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 169         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 170                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
 171                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 172                                                  obj_type) << 16,
 173                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 174                                                  obj_id));
 175                 break;
 176         case MLX5_CMD_OP_QUERY_MKEY:
 177                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 178                                         MLX5_GET(query_mkey_in, in,
 179                                                  mkey_index));
 180                 break;
 181         case MLX5_CMD_OP_QUERY_CQ:
 182                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 183                                         MLX5_GET(query_cq_in, in, cqn));
 184                 break;
 185         case MLX5_CMD_OP_MODIFY_CQ:
 186                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 187                                         MLX5_GET(modify_cq_in, in, cqn));
 188                 break;
 189         case MLX5_CMD_OP_QUERY_SQ:
 190                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 191                                         MLX5_GET(query_sq_in, in, sqn));
 192                 break;
 193         case MLX5_CMD_OP_MODIFY_SQ:
 194                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 195                                         MLX5_GET(modify_sq_in, in, sqn));
 196                 break;
 197         case MLX5_CMD_OP_QUERY_RQ:
 198                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 199                                         MLX5_GET(query_rq_in, in, rqn));
 200                 break;
 201         case MLX5_CMD_OP_MODIFY_RQ:
 202                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 203                                         MLX5_GET(modify_rq_in, in, rqn));
 204                 break;
 205         case MLX5_CMD_OP_QUERY_RMP:
 206                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 207                                         MLX5_GET(query_rmp_in, in, rmpn));
 208                 break;
 209         case MLX5_CMD_OP_MODIFY_RMP:
 210                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 211                                         MLX5_GET(modify_rmp_in, in, rmpn));
 212                 break;
 213         case MLX5_CMD_OP_QUERY_RQT:
 214                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 215                                         MLX5_GET(query_rqt_in, in, rqtn));
 216                 break;
 217         case MLX5_CMD_OP_MODIFY_RQT:
 218                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 219                                         MLX5_GET(modify_rqt_in, in, rqtn));
 220                 break;
 221         case MLX5_CMD_OP_QUERY_TIR:
 222                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 223                                         MLX5_GET(query_tir_in, in, tirn));
 224                 break;
 225         case MLX5_CMD_OP_MODIFY_TIR:
 226                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 227                                         MLX5_GET(modify_tir_in, in, tirn));
 228                 break;
 229         case MLX5_CMD_OP_QUERY_TIS:
 230                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 231                                         MLX5_GET(query_tis_in, in, tisn));
 232                 break;
 233         case MLX5_CMD_OP_MODIFY_TIS:
 234                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 235                                         MLX5_GET(modify_tis_in, in, tisn));
 236                 break;
 237         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 238                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 239                                         MLX5_GET(query_flow_table_in, in,
 240                                                  table_id));
 241                 break;
 242         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 243                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 244                                         MLX5_GET(modify_flow_table_in, in,
 245                                                  table_id));
 246                 break;
 247         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 248                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 249                                         MLX5_GET(query_flow_group_in, in,
 250                                                  group_id));
 251                 break;
 252         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 253                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 254                                         MLX5_GET(query_fte_in, in,
 255                                                  flow_index));
 256                 break;
 257         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 258                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 259                                         MLX5_GET(set_fte_in, in, flow_index));
 260                 break;
 261         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 262                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 263                                         MLX5_GET(query_q_counter_in, in,
 264                                                  counter_set_id));
 265                 break;
 266         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 267                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 268                                         MLX5_GET(query_flow_counter_in, in,
 269                                                  flow_counter_id));
 270                 break;
 271         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 272                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 273                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 274                                                  obj_id));
 275                 break;
 276         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 277                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 278                                         MLX5_GET(query_scheduling_element_in,
 279                                                  in, scheduling_element_id));
 280                 break;
 281         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 282                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 283                                         MLX5_GET(modify_scheduling_element_in,
 284                                                  in, scheduling_element_id));
 285                 break;
 286         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 287                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 288                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 289                                                  vxlan_udp_port));
 290                 break;
 291         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 292                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 293                                         MLX5_GET(query_l2_table_entry_in, in,
 294                                                  table_index));
 295                 break;
 296         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 297                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 298                                         MLX5_GET(set_l2_table_entry_in, in,
 299                                                  table_index));
 300                 break;
 301         case MLX5_CMD_OP_QUERY_QP:
 302                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 303                                         MLX5_GET(query_qp_in, in, qpn));
 304                 break;
 305         case MLX5_CMD_OP_RST2INIT_QP:
 306                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 307                                         MLX5_GET(rst2init_qp_in, in, qpn));
 308                 break;
 309         case MLX5_CMD_OP_INIT2RTR_QP:
 310                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 311                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 312                 break;
 313         case MLX5_CMD_OP_RTR2RTS_QP:
 314                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 315                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 316                 break;
 317         case MLX5_CMD_OP_RTS2RTS_QP:
 318                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 319                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 320                 break;
 321         case MLX5_CMD_OP_SQERR2RTS_QP:
 322                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 323                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 324                 break;
 325         case MLX5_CMD_OP_2ERR_QP:
 326                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 327                                         MLX5_GET(qp_2err_in, in, qpn));
 328                 break;
 329         case MLX5_CMD_OP_2RST_QP:
 330                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 331                                         MLX5_GET(qp_2rst_in, in, qpn));
 332                 break;
 333         case MLX5_CMD_OP_QUERY_DCT:
 334                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 335                                         MLX5_GET(query_dct_in, in, dctn));
 336                 break;
 337         case MLX5_CMD_OP_QUERY_XRQ:
 338         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 339         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 340                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 341                                         MLX5_GET(query_xrq_in, in, xrqn));
 342                 break;
 343         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 344                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 345                                         MLX5_GET(query_xrc_srq_in, in,
 346                                                  xrc_srqn));
 347                 break;
 348         case MLX5_CMD_OP_ARM_XRC_SRQ:
 349                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 350                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 351                 break;
 352         case MLX5_CMD_OP_QUERY_SRQ:
 353                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 354                                         MLX5_GET(query_srq_in, in, srqn));
 355                 break;
 356         case MLX5_CMD_OP_ARM_RQ:
 357                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 358                                         MLX5_GET(arm_rq_in, in, srq_number));
 359                 break;
 360         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 361                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 362                                         MLX5_GET(drain_dct_in, in, dctn));
 363                 break;
 364         case MLX5_CMD_OP_ARM_XRQ:
 365         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 366                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 367                                         MLX5_GET(arm_xrq_in, in, xrqn));
 368                 break;
 369         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 370                 obj_id = get_enc_obj_id
 371                                 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 372                                  MLX5_GET(query_packet_reformat_context_in,
 373                                           in, packet_reformat_id));
 374                 break;
 375         default:
 376                 obj_id = 0;
 377         }
 378
 379         return obj_id;
 380 }
 381
 382 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 383                                  struct ib_uobject *uobj, const void *in)
 384 {
 385         struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 386         u64 obj_id = devx_get_obj_id(in);
 387
 388         if (!obj_id)
 389                 return false;
 390
 391         switch (uobj_get_object_id(uobj)) {
 392         case UVERBS_OBJECT_CQ:
 393                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 394                                       to_mcq(uobj->object)->mcq.cqn) ==
 395                                       obj_id;
 396
 397         case UVERBS_OBJECT_SRQ:
 398         {
 399                 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 400                 u16 opcode;
 401
 402                 switch (srq->common.res) {
 403                 case MLX5_RES_XSRQ:
 404                         opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 405                         break;
 406                 case MLX5_RES_XRQ:
 407                         opcode = MLX5_CMD_OP_CREATE_XRQ;
 408                         break;
 409                 default:
 410                         if (!dev->mdev->issi)
 411                                 opcode = MLX5_CMD_OP_CREATE_SRQ;
 412                         else
 413                                 opcode = MLX5_CMD_OP_CREATE_RMP;
 414                 }
 415
 416                 return get_enc_obj_id(opcode,
 417                                       to_msrq(uobj->object)->msrq.srqn) ==
 418                                       obj_id;
 419         }
 420
 421         case UVERBS_OBJECT_QP:
 422         {
 423                 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 424                 enum ib_qp_type qp_type = qp->ibqp.qp_type;
 425
 426                 if (qp_type == IB_QPT_RAW_PACKET ||
 427                     (qp->flags & MLX5_IB_QP_UNDERLAY)) {
 428                         struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 429                                                          &qp->raw_packet_qp;
 430                         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 431                         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 432
 433                         return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 434                                                rq->base.mqp.qpn) == obj_id ||
 435                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 436                                                sq->base.mqp.qpn) == obj_id ||
 437                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 438                                                rq->tirn) == obj_id ||
 439                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 440                                                sq->tisn) == obj_id);
 441                 }
 442
 443                 if (qp_type == MLX5_IB_QPT_DCT)
 444                         return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 445                                               qp->dct.mdct.mqp.qpn) == obj_id;
 446
 447                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 448                                       qp->ibqp.qp_num) == obj_id;
 449         }
 450
 451         case UVERBS_OBJECT_WQ:
 452                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 453                                       to_mrwq(uobj->object)->core_qp.qpn) ==
 454                                       obj_id;
 455
 456         case UVERBS_OBJECT_RWQ_IND_TBL:
 457                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 458                                       to_mrwq_ind_table(uobj->object)->rqtn) ==
 459                                       obj_id;
 460
 461         case MLX5_IB_OBJECT_DEVX_OBJ:
 462                 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
 463
 464         default:
 465                 return false;
 466         }
 467 }
 468
 469 static void devx_set_umem_valid(const void *in)
 470 {
 471         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 472
 473         switch (opcode) {
 474         case MLX5_CMD_OP_CREATE_MKEY:
 475                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 476                 break;
 477         case MLX5_CMD_OP_CREATE_CQ:
 478         {
 479                 void *cqc;
 480
 481                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 482                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 483                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 484                 break;
 485         }
 486         case MLX5_CMD_OP_CREATE_QP:
 487         {
 488                 void *qpc;
 489
 490                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 491                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 492                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 493                 break;
 494         }
 495
 496         case MLX5_CMD_OP_CREATE_RQ:
 497         {
 498                 void *rqc, *wq;
 499
 500                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 501                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 502                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 503                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 504                 break;
 505         }
 506
 507         case MLX5_CMD_OP_CREATE_SQ:
 508         {
 509                 void *sqc, *wq;
 510
 511                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 512                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 513                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 514                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 515                 break;
 516         }
 517
 518         case MLX5_CMD_OP_MODIFY_CQ:
 519                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 520                 break;
 521
 522         case MLX5_CMD_OP_CREATE_RMP:
 523         {
 524                 void *rmpc, *wq;
 525
 526                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 527                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 528                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 529                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 530                 break;
 531         }
 532
 533         case MLX5_CMD_OP_CREATE_XRQ:
 534         {
 535                 void *xrqc, *wq;
 536
 537                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 538                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 539                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 540                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 541                 break;
 542         }
 543
 544         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 545         {
 546                 void *xrc_srqc;
 547
 548                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 549                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 550                                         xrc_srq_context_entry);
 551                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 552                 break;
 553         }
 554
 555         default:
 556                 return;
 557         }
 558 }
 559
 560 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 561 {
 562         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 563
 564         switch (*opcode) {
 565         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 566         case MLX5_CMD_OP_CREATE_MKEY:
 567         case MLX5_CMD_OP_CREATE_CQ:
 568         case MLX5_CMD_OP_ALLOC_PD:
 569         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 570         case MLX5_CMD_OP_CREATE_RMP:
 571         case MLX5_CMD_OP_CREATE_SQ:
 572         case MLX5_CMD_OP_CREATE_RQ:
 573         case MLX5_CMD_OP_CREATE_RQT:
 574         case MLX5_CMD_OP_CREATE_TIR:
 575         case MLX5_CMD_OP_CREATE_TIS:
 576         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 577         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 578         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 579         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 580         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 581         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 582         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 583         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 584         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 585         case MLX5_CMD_OP_CREATE_QP:
 586         case MLX5_CMD_OP_CREATE_SRQ:
 587         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 588         case MLX5_CMD_OP_CREATE_DCT:
 589         case MLX5_CMD_OP_CREATE_XRQ:
 590         case MLX5_CMD_OP_ATTACH_TO_MCG:
 591         case MLX5_CMD_OP_ALLOC_XRCD:
 592                 return true;
 593         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 594         {
 595                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 596                 if (op_mod == 0)
 597                         return true;
 598                 return false;
 599         }
 600         default:
 601                 return false;
 602         }
 603 }
 604
 605 static bool devx_is_obj_modify_cmd(const void *in)
 606 {
 607         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 608
 609         switch (opcode) {
 610         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 611         case MLX5_CMD_OP_MODIFY_CQ:
 612         case MLX5_CMD_OP_MODIFY_RMP:
 613         case MLX5_CMD_OP_MODIFY_SQ:
 614         case MLX5_CMD_OP_MODIFY_RQ:
 615         case MLX5_CMD_OP_MODIFY_RQT:
 616         case MLX5_CMD_OP_MODIFY_TIR:
 617         case MLX5_CMD_OP_MODIFY_TIS:
 618         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 619         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 620         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 621         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 622         case MLX5_CMD_OP_RST2INIT_QP:
 623         case MLX5_CMD_OP_INIT2RTR_QP:
 624         case MLX5_CMD_OP_RTR2RTS_QP:
 625         case MLX5_CMD_OP_RTS2RTS_QP:
 626         case MLX5_CMD_OP_SQERR2RTS_QP:
 627         case MLX5_CMD_OP_2ERR_QP:
 628         case MLX5_CMD_OP_2RST_QP:
 629         case MLX5_CMD_OP_ARM_XRC_SRQ:
 630         case MLX5_CMD_OP_ARM_RQ:
 631         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 632         case MLX5_CMD_OP_ARM_XRQ:
 633         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 634                 return true;
 635         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 636         {
 637                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 638
 639                 if (op_mod == 1)
 640                         return true;
 641                 return false;
 642         }
 643         default:
 644                 return false;
 645         }
 646 }
 647
 648 static bool devx_is_obj_query_cmd(const void *in)
 649 {
 650         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 651
 652         switch (opcode) {
 653         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 654         case MLX5_CMD_OP_QUERY_MKEY:
 655         case MLX5_CMD_OP_QUERY_CQ:
 656         case MLX5_CMD_OP_QUERY_RMP:
 657         case MLX5_CMD_OP_QUERY_SQ:
 658         case MLX5_CMD_OP_QUERY_RQ:
 659         case MLX5_CMD_OP_QUERY_RQT:
 660         case MLX5_CMD_OP_QUERY_TIR:
 661         case MLX5_CMD_OP_QUERY_TIS:
 662         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 663         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 664         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 665         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 666         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 667         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 668         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 669         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 670         case MLX5_CMD_OP_QUERY_QP:
 671         case MLX5_CMD_OP_QUERY_SRQ:
 672         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 673         case MLX5_CMD_OP_QUERY_DCT:
 674         case MLX5_CMD_OP_QUERY_XRQ:
 675         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 676         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 677         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 678                 return true;
 679         default:
 680                 return false;
 681         }
 682 }
 683
 684 static bool devx_is_whitelist_cmd(void *in)
 685 {
 686         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 687
 688         switch (opcode) {
 689         case MLX5_CMD_OP_QUERY_HCA_CAP:
 690         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 691         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 692                 return true;
 693         default:
 694                 return false;
 695         }
 696 }
 697
 698 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 699 {
 700         if (devx_is_whitelist_cmd(cmd_in)) {
 701                 struct mlx5_ib_dev *dev;
 702
 703                 if (c->devx_uid)
 704                         return c->devx_uid;
 705
 706                 dev = to_mdev(c->ibucontext.device);
 707                 if (dev->devx_whitelist_uid)
 708                         return dev->devx_whitelist_uid;
 709
 710                 return -EOPNOTSUPP;
 711         }
 712
 713         if (!c->devx_uid)
 714                 return -EINVAL;
 715
 716         return c->devx_uid;
 717 }
 718 static bool devx_is_general_cmd(void *in)
 719 {
 720         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 721
 722         if (opcode >= MLX5_CMD_OP_GENERAL_START &&
 723             opcode < MLX5_CMD_OP_GENERAL_END)
 724                 return true;
 725
 726         switch (opcode) {
 727         case MLX5_CMD_OP_QUERY_HCA_CAP:
 728         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 729         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 730         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 731         case MLX5_CMD_OP_QUERY_ADAPTER:
 732         case MLX5_CMD_OP_QUERY_ISSI:
 733         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 734         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 735         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 736         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 737         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 738         case MLX5_CMD_OP_NOP:
 739         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 740         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 741         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 742                 return true;
 743         default:
 744                 return false;
 745         }
 746 }
 747
 748 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 749         struct uverbs_attr_bundle *attrs)
 750 {
 751         struct mlx5_ib_ucontext *c;
 752         struct mlx5_ib_dev *dev;
 753         int user_vector;
 754         int dev_eqn;
 755         unsigned int irqn;
 756         int err;
 757
 758         if (uverbs_copy_from(&user_vector, attrs,
 759                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 760                 return -EFAULT;
 761
 762         c = devx_ufile2uctx(attrs);
 763         if (IS_ERR(c))
 764                 return PTR_ERR(c);
 765         dev = to_mdev(c->ibucontext.device);
 766
 767         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
 768         if (err < 0)
 769                 return err;
 770
 771         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 772                            &dev_eqn, sizeof(dev_eqn)))
 773                 return -EFAULT;
 774
 775         return 0;
 776 }
 777
 778 /*
 779  *Security note:
 780  * The hardware protection mechanism works like this: Each device object that
 781  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
 782  * the device specification manual) upon its creation. Then upon doorbell,
 783  * hardware fetches the object context for which the doorbell was rang, and
 784  * validates that the UAR through which the DB was rang matches the UAR ID
 785  * of the object.
 786  * If no match the doorbell is silently ignored by the hardware. Of course,
 787  * the user cannot ring a doorbell on a UAR that was not mapped to it.
 788  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
 789  * mailboxes (except tagging them with UID), we expose to the user its UAR
 790  * ID, so it can embed it in these objects in the expected specification
 791  * format. So the only thing the user can do is hurt itself by creating a
 792  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
 793  * may ring a doorbell on its objects.
 794  * The consequence of that will be that another user can schedule a QP/SQ
 795  * of the buggy user for execution (just insert it to the hardware schedule
 796  * queue or arm its CQ for event generation), no further harm is expected.
 797  */
 798 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
 799         struct uverbs_attr_bundle *attrs)
 800 {
 801         struct mlx5_ib_ucontext *c;
 802         struct mlx5_ib_dev *dev;
 803         u32 user_idx;
 804         s32 dev_idx;
 805
 806         c = devx_ufile2uctx(attrs);
 807         if (IS_ERR(c))
 808                 return PTR_ERR(c);
 809         dev = to_mdev(c->ibucontext.device);
 810
 811         if (uverbs_copy_from(&user_idx, attrs,
 812                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
 813                 return -EFAULT;
 814
 815         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
 816         if (dev_idx < 0)
 817                 return dev_idx;
 818
 819         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
 820                            &dev_idx, sizeof(dev_idx)))
 821                 return -EFAULT;
 822
 823         return 0;
 824 }
 825
 826 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
 827         struct uverbs_attr_bundle *attrs)
 828 {
 829         struct mlx5_ib_ucontext *c;
 830         struct mlx5_ib_dev *dev;
 831         void *cmd_in = uverbs_attr_get_alloced_ptr(
 832                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
 833         int cmd_out_len = uverbs_attr_get_len(attrs,
 834                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
 835         void *cmd_out;
 836         int err;
 837         int uid;
 838
 839         c = devx_ufile2uctx(attrs);
 840         if (IS_ERR(c))
 841                 return PTR_ERR(c);
 842         dev = to_mdev(c->ibucontext.device);
 843
 844         uid = devx_get_uid(c, cmd_in);
 845         if (uid < 0)
 846                 return uid;
 847
 848         /* Only white list of some general HCA commands are allowed for this method. */
 849         if (!devx_is_general_cmd(cmd_in))
 850                 return -EINVAL;
 851
 852         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
 853         if (IS_ERR(cmd_out))
 854                 return PTR_ERR(cmd_out);
 855
 856         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
 857         err = mlx5_cmd_exec(dev->mdev, cmd_in,
 858                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
 859                             cmd_out, cmd_out_len);
 860         if (err)
 861                 return err;
 862
 863         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
 864                               cmd_out_len);
 865 }
 866
 867 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
 868                                        u32 *dinlen,
 869                                        u32 *obj_id)
 870 {
 871         u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
 872         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
 873
 874         *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 875         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
 876
 877         MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
 878         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
 879
 880         switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
 881         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 882                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
 883                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
 884                 break;
 885
 886         case MLX5_CMD_OP_CREATE_UMEM:
 887                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 888                          MLX5_CMD_OP_DESTROY_UMEM);
 889                 break;
 890         case MLX5_CMD_OP_CREATE_MKEY:
 891                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
 892                 break;
 893         case MLX5_CMD_OP_CREATE_CQ:
 894                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
 895                 break;
 896         case MLX5_CMD_OP_ALLOC_PD:
 897                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
 898                 break;
 899         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 900                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 901                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 902                 break;
 903         case MLX5_CMD_OP_CREATE_RMP:
 904                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
 905                 break;
 906         case MLX5_CMD_OP_CREATE_SQ:
 907                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
 908                 break;
 909         case MLX5_CMD_OP_CREATE_RQ:
 910                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
 911                 break;
 912         case MLX5_CMD_OP_CREATE_RQT:
 913                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
 914                 break;
 915         case MLX5_CMD_OP_CREATE_TIR:
 916                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
 917                 break;
 918         case MLX5_CMD_OP_CREATE_TIS:
 919                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
 920                 break;
 921         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 922                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 923                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 924                 break;
 925         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 926                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
 927                 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
 928                 MLX5_SET(destroy_flow_table_in, din, other_vport,
 929                          MLX5_GET(create_flow_table_in,  in, other_vport));
 930                 MLX5_SET(destroy_flow_table_in, din, vport_number,
 931                          MLX5_GET(create_flow_table_in,  in, vport_number));
 932                 MLX5_SET(destroy_flow_table_in, din, table_type,
 933                          MLX5_GET(create_flow_table_in,  in, table_type));
 934                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
 935                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 936                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
 937                 break;
 938         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 939                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
 940                 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
 941                 MLX5_SET(destroy_flow_group_in, din, other_vport,
 942                          MLX5_GET(create_flow_group_in, in, other_vport));
 943                 MLX5_SET(destroy_flow_group_in, din, vport_number,
 944                          MLX5_GET(create_flow_group_in, in, vport_number));
 945                 MLX5_SET(destroy_flow_group_in, din, table_type,
 946                          MLX5_GET(create_flow_group_in, in, table_type));
 947                 MLX5_SET(destroy_flow_group_in, din, table_id,
 948                          MLX5_GET(create_flow_group_in, in, table_id));
 949                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
 950                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 951                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
 952                 break;
 953         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 954                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
 955                 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
 956                 MLX5_SET(delete_fte_in, din, other_vport,
 957                          MLX5_GET(set_fte_in,  in, other_vport));
 958                 MLX5_SET(delete_fte_in, din, vport_number,
 959                          MLX5_GET(set_fte_in, in, vport_number));
 960                 MLX5_SET(delete_fte_in, din, table_type,
 961                          MLX5_GET(set_fte_in, in, table_type));
 962                 MLX5_SET(delete_fte_in, din, table_id,
 963                          MLX5_GET(set_fte_in, in, table_id));
 964                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
 965                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 966                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
 967                 break;
 968         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 969                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 970                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
 971                 break;
 972         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 973                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 974                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
 975                 break;
 976         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 977                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 978                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
 979                 break;
 980         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 981                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
 982                 *obj_id = MLX5_GET(create_scheduling_element_out, out,
 983                                    scheduling_element_id);
 984                 MLX5_SET(destroy_scheduling_element_in, din,
 985                          scheduling_hierarchy,
 986                          MLX5_GET(create_scheduling_element_in, in,
 987                                   scheduling_hierarchy));
 988                 MLX5_SET(destroy_scheduling_element_in, din,
 989                          scheduling_element_id, *obj_id);
 990                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 991                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
 992                 break;
 993         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 994                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
 995                 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
 996                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
 997                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 998                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
 999                 break;
1000         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1001                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1002                 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1003                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1004                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1005                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1006                 break;
1007         case MLX5_CMD_OP_CREATE_QP:
1008                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1009                 break;
1010         case MLX5_CMD_OP_CREATE_SRQ:
1011                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1012                 break;
1013         case MLX5_CMD_OP_CREATE_XRC_SRQ:
1014                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1015                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
1016                 break;
1017         case MLX5_CMD_OP_CREATE_DCT:
1018                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1019                 break;
1020         case MLX5_CMD_OP_CREATE_XRQ:
1021                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1022                 break;
1023         case MLX5_CMD_OP_ATTACH_TO_MCG:
1024                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1025                 MLX5_SET(detach_from_mcg_in, din, qpn,
1026                          MLX5_GET(attach_to_mcg_in, in, qpn));
1027                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1028                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1029                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1030                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1031                 break;
1032         case MLX5_CMD_OP_ALLOC_XRCD:
1033                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1034                 break;
1035         default:
1036                 /* The entry must match to one of the devx_is_obj_create_cmd */
1037                 WARN_ON(true);
1038                 break;
1039         }
1040 }
1041
1042 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1043                                      struct mlx5_ib_dev *dev,
1044                                      void *in, void *out)
1045 {
1046         struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
1047         struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1048         unsigned long flags;
1049         struct mlx5_core_mkey *mkey;
1050         void *mkc;
1051         u8 key;
1052         int err;
1053
1054         mkey = &devx_mr->mmkey;
1055         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1056         key = MLX5_GET(mkc, mkc, mkey_7_0);
1057         mkey->key = mlx5_idx_to_mkey(
1058                         MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1059         mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1060         mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1061         mkey->size = MLX5_GET64(mkc, mkc, len);
1062         mkey->pd = MLX5_GET(mkc, mkc, pd);
1063         devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1064
1065         write_lock_irqsave(&table->lock, flags);
1066         err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key),
1067                                 mkey);
1068         write_unlock_irqrestore(&table->lock, flags);
1069         return err;
1070 }
1071
1072 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1073                                    struct devx_obj *obj,
1074                                    void *in, int in_len)
1075 {
1076         int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1077                         MLX5_FLD_SZ_BYTES(create_mkey_in,
1078                         memory_key_mkey_entry);
1079         void *mkc;
1080         u8 access_mode;
1081
1082         if (in_len < min_len)
1083                 return -EINVAL;
1084
1085         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1086
1087         access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1088         access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1089
1090         if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1091                 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1092                 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1093                         obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1094                 return 0;
1095         }
1096
1097         MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1098         return 0;
1099 }
1100
1101 static void devx_free_indirect_mkey(struct rcu_head *rcu)
1102 {
1103         kfree(container_of(rcu, struct devx_obj, devx_mr.rcu));
1104 }
1105
1106 /* This function to delete from the radix tree needs to be called before
1107  * destroying the underlying mkey. Otherwise a race might occur in case that
1108  * other thread will get the same mkey before this one will be deleted,
1109  * in that case it will fail via inserting to the tree its own data.
1110  *
1111  * Note:
1112  * An error in the destroy is not expected unless there is some other indirect
1113  * mkey which points to this one. In a kernel cleanup flow it will be just
1114  * destroyed in the iterative destruction call. In a user flow, in case
1115  * the application didn't close in the expected order it's its own problem,
1116  * the mkey won't be part of the tree, in both cases the kernel is safe.
1117  */
1118 static void devx_cleanup_mkey(struct devx_obj *obj)
1119 {
1120         struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table;
1121         unsigned long flags;
1122
1123         write_lock_irqsave(&table->lock, flags);
1124         radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key));
1125         write_unlock_irqrestore(&table->lock, flags);
1126 }
1127
1128 static int devx_obj_cleanup(struct ib_uobject *uobject,
1129                             enum rdma_remove_reason why,
1130                             struct uverbs_attr_bundle *attrs)
1131 {
1132         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1133         struct devx_obj *obj = uobject->object;
1134         int ret;
1135
1136         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY)
1137                 devx_cleanup_mkey(obj);
1138
1139         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1140                 ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
1141         else
1142                 ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
1143                                     sizeof(out));
1144         if (ib_is_destroy_retryable(ret, why, uobject))
1145                 return ret;
1146
1147         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1148                 struct mlx5_ib_dev *dev =
1149                         mlx5_udata_to_mdev(&attrs->driver_udata);
1150
1151                 call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu,
1152                           devx_free_indirect_mkey);
1153                 return ret;
1154         }
1155
1156         kfree(obj);
1157         return ret;
1158 }
1159
1160 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1161         struct uverbs_attr_bundle *attrs)
1162 {
1163         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1164         int cmd_out_len =  uverbs_attr_get_len(attrs,
1165                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1166         int cmd_in_len = uverbs_attr_get_len(attrs,
1167                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1168         void *cmd_out;
1169         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1170                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1171         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1172                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1173         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1174         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1175         struct devx_obj *obj;
1176         u16 obj_type = 0;
1177         int err;
1178         int uid;
1179         u32 obj_id;
1180         u16 opcode;
1181
1182         uid = devx_get_uid(c, cmd_in);
1183         if (uid < 0)
1184                 return uid;
1185
1186         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1187                 return -EINVAL;
1188
1189         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1190         if (IS_ERR(cmd_out))
1191                 return PTR_ERR(cmd_out);
1192
1193         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1194         if (!obj)
1195                 return -ENOMEM;
1196
1197         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1198         if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1199                 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1200                 if (err)
1201                         goto obj_free;
1202         } else {
1203                 devx_set_umem_valid(cmd_in);
1204         }
1205
1206         if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1207                 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1208                 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct,
1209                                            cmd_in, cmd_in_len,
1210                                            cmd_out, cmd_out_len);
1211         } else {
1212                 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1213                                     cmd_in_len,
1214                                     cmd_out, cmd_out_len);
1215         }
1216
1217         if (err)
1218                 goto obj_free;
1219
1220         uobj->object = obj;
1221         obj->mdev = dev->mdev;
1222         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1223                                    &obj_id);
1224         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1225
1226         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1227                 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1228                 if (err)
1229                         goto obj_destroy;
1230         }
1231
1232         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1233         if (err)
1234                 goto err_copy;
1235
1236         if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1237                 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1238
1239         obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1240
1241         return 0;
1242
1243 err_copy:
1244         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY)
1245                 devx_cleanup_mkey(obj);
1246 obj_destroy:
1247         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1248                 mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
1249         else
1250                 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
1251                               sizeof(out));
1252 obj_free:
1253         kfree(obj);
1254         return err;
1255 }
1256
1257 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1258         struct uverbs_attr_bundle *attrs)
1259 {
1260         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1261         int cmd_out_len = uverbs_attr_get_len(attrs,
1262                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1263         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1264                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1265         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1266                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1267         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1268         void *cmd_out;
1269         int err;
1270         int uid;
1271
1272         uid = devx_get_uid(c, cmd_in);
1273         if (uid < 0)
1274                 return uid;
1275
1276         if (!devx_is_obj_modify_cmd(cmd_in))
1277                 return -EINVAL;
1278
1279         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1280                 return -EINVAL;
1281
1282         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1283         if (IS_ERR(cmd_out))
1284                 return PTR_ERR(cmd_out);
1285
1286         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1287         devx_set_umem_valid(cmd_in);
1288
1289         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1290                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1291                             cmd_out, cmd_out_len);
1292         if (err)
1293                 return err;
1294
1295         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1296                               cmd_out, cmd_out_len);
1297 }
1298
1299 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1300         struct uverbs_attr_bundle *attrs)
1301 {
1302         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1303         int cmd_out_len = uverbs_attr_get_len(attrs,
1304                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1305         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1306                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1307         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1308                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1309         void *cmd_out;
1310         int err;
1311         int uid;
1312         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1313
1314         uid = devx_get_uid(c, cmd_in);
1315         if (uid < 0)
1316                 return uid;
1317
1318         if (!devx_is_obj_query_cmd(cmd_in))
1319                 return -EINVAL;
1320
1321         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1322                 return -EINVAL;
1323
1324         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1325         if (IS_ERR(cmd_out))
1326                 return PTR_ERR(cmd_out);
1327
1328         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1329         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1330                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1331                             cmd_out, cmd_out_len);
1332         if (err)
1333                 return err;
1334
1335         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1336                               cmd_out, cmd_out_len);
1337 }
1338
1339 struct devx_async_event_queue {
1340         spinlock_t              lock;
1341         wait_queue_head_t       poll_wait;
1342         struct list_head        event_list;
1343         atomic_t                bytes_in_use;
1344         u8                      is_destroyed:1;
1345 };
1346
1347 struct devx_async_cmd_event_file {
1348         struct ib_uobject               uobj;
1349         struct devx_async_event_queue   ev_queue;
1350         struct mlx5_async_ctx           async_ctx;
1351 };
1352
1353 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1354 {
1355         spin_lock_init(&ev_queue->lock);
1356         INIT_LIST_HEAD(&ev_queue->event_list);
1357         init_waitqueue_head(&ev_queue->poll_wait);
1358         atomic_set(&ev_queue->bytes_in_use, 0);
1359         ev_queue->is_destroyed = 0;
1360 }
1361
1362 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1363         struct uverbs_attr_bundle *attrs)
1364 {
1365         struct devx_async_cmd_event_file *ev_file;
1366
1367         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1368                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1369         struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1370
1371         ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1372                                uobj);
1373         devx_init_event_queue(&ev_file->ev_queue);
1374         mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1375         return 0;
1376 }
1377
1378 static void devx_query_callback(int status, struct mlx5_async_work *context)
1379 {
1380         struct devx_async_data *async_data =
1381                 container_of(context, struct devx_async_data, cb_work);
1382         struct ib_uobject *fd_uobj = async_data->fd_uobj;
1383         struct devx_async_cmd_event_file *ev_file;
1384         struct devx_async_event_queue *ev_queue;
1385         unsigned long flags;
1386
1387         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1388                                uobj);
1389         ev_queue = &ev_file->ev_queue;
1390
1391         spin_lock_irqsave(&ev_queue->lock, flags);
1392         list_add_tail(&async_data->list, &ev_queue->event_list);
1393         spin_unlock_irqrestore(&ev_queue->lock, flags);
1394
1395         wake_up_interruptible(&ev_queue->poll_wait);
1396         fput(fd_uobj->object);
1397 }
1398
1399 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1400
1401 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1402         struct uverbs_attr_bundle *attrs)
1403 {
1404         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1405                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1406         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1407                                 attrs,
1408                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1409         u16 cmd_out_len;
1410         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1411                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1412         struct ib_uobject *fd_uobj;
1413         int err;
1414         int uid;
1415         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1416         struct devx_async_cmd_event_file *ev_file;
1417         struct devx_async_data *async_data;
1418
1419         uid = devx_get_uid(c, cmd_in);
1420         if (uid < 0)
1421                 return uid;
1422
1423         if (!devx_is_obj_query_cmd(cmd_in))
1424                 return -EINVAL;
1425
1426         err = uverbs_get_const(&cmd_out_len, attrs,
1427                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1428         if (err)
1429                 return err;
1430
1431         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1432                 return -EINVAL;
1433
1434         fd_uobj = uverbs_attr_get_uobject(attrs,
1435                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1436         if (IS_ERR(fd_uobj))
1437                 return PTR_ERR(fd_uobj);
1438
1439         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1440                                uobj);
1441
1442         if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1443                         MAX_ASYNC_BYTES_IN_USE) {
1444                 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1445                 return -EAGAIN;
1446         }
1447
1448         async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1449                                           cmd_out_len), GFP_KERNEL);
1450         if (!async_data) {
1451                 err = -ENOMEM;
1452                 goto sub_bytes;
1453         }
1454
1455         err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1456                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1457         if (err)
1458                 goto free_async;
1459
1460         async_data->cmd_out_len = cmd_out_len;
1461         async_data->mdev = mdev;
1462         async_data->fd_uobj = fd_uobj;
1463
1464         get_file(fd_uobj->object);
1465         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1466         err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1467                     uverbs_attr_get_len(attrs,
1468                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1469                     async_data->hdr.out_data,
1470                     async_data->cmd_out_len,
1471                     devx_query_callback, &async_data->cb_work);
1472
1473         if (err)
1474                 goto cb_err;
1475
1476         return 0;
1477
1478 cb_err:
1479         fput(fd_uobj->object);
1480 free_async:
1481         kvfree(async_data);
1482 sub_bytes:
1483         atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1484         return err;
1485 }
1486
1487 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
1488                          struct uverbs_attr_bundle *attrs,
1489                          struct devx_umem *obj)
1490 {
1491         u64 addr;
1492         size_t size;
1493         u32 access;
1494         int npages;
1495         int err;
1496         u32 page_mask;
1497
1498         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
1499             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
1500                 return -EFAULT;
1501
1502         err = uverbs_get_flags32(&access, attrs,
1503                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1504                                  IB_ACCESS_LOCAL_WRITE |
1505                                  IB_ACCESS_REMOTE_WRITE |
1506                                  IB_ACCESS_REMOTE_READ);
1507         if (err)
1508                 return err;
1509
1510         err = ib_check_mr_access(access);
1511         if (err)
1512                 return err;
1513
1514         obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0);
1515         if (IS_ERR(obj->umem))
1516                 return PTR_ERR(obj->umem);
1517
1518         mlx5_ib_cont_pages(obj->umem, obj->umem->address,
1519                            MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
1520                            &obj->page_shift, &obj->ncont, NULL);
1521
1522         if (!npages) {
1523                 ib_umem_release(obj->umem);
1524                 return -EINVAL;
1525         }
1526
1527         page_mask = (1 << obj->page_shift) - 1;
1528         obj->page_offset = obj->umem->address & page_mask;
1529
1530         return 0;
1531 }
1532
1533 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
1534                                    struct devx_umem *obj,
1535                                    struct devx_umem_reg_cmd *cmd)
1536 {
1537         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
1538                     (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
1539         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
1540         return PTR_ERR_OR_ZERO(cmd->in);
1541 }
1542
1543 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
1544                                     struct devx_umem *obj,
1545                                     struct devx_umem_reg_cmd *cmd)
1546 {
1547         void *umem;
1548         __be64 *mtt;
1549
1550         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
1551         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
1552
1553         MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
1554         MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
1555         MLX5_SET(umem, umem, log_page_size, obj->page_shift -
1556                                             MLX5_ADAPTER_PAGE_SHIFT);
1557         MLX5_SET(umem, umem, page_offset, obj->page_offset);
1558         mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
1559                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
1560                              MLX5_IB_MTT_READ);
1561 }
1562
1563 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
1564         struct uverbs_attr_bundle *attrs)
1565 {
1566         struct devx_umem_reg_cmd cmd;
1567         struct devx_umem *obj;
1568         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1569                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
1570         u32 obj_id;
1571         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1572                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1573         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1574         int err;
1575
1576         if (!c->devx_uid)
1577                 return -EINVAL;
1578
1579         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
1580         if (!obj)
1581                 return -ENOMEM;
1582
1583         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
1584         if (err)
1585                 goto err_obj_free;
1586
1587         err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
1588         if (err)
1589                 goto err_umem_release;
1590
1591         devx_umem_reg_cmd_build(dev, obj, &cmd);
1592
1593         MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
1594         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
1595                             sizeof(cmd.out));
1596         if (err)
1597                 goto err_umem_release;
1598
1599         obj->mdev = dev->mdev;
1600         uobj->object = obj;
1601         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
1602         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
1603         if (err)
1604                 goto err_umem_destroy;
1605
1606         return 0;
1607
1608 err_umem_destroy:
1609         mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
1610 err_umem_release:
1611         ib_umem_release(obj->umem);
1612 err_obj_free:
1613         kfree(obj);
1614         return err;
1615 }
1616
1617 static int devx_umem_cleanup(struct ib_uobject *uobject,
1618                              enum rdma_remove_reason why,
1619                              struct uverbs_attr_bundle *attrs)
1620 {
1621         struct devx_umem *obj = uobject->object;
1622         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1623         int err;
1624
1625         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
1626         if (ib_is_destroy_retryable(err, why, uobject))
1627                 return err;
1628
1629         ib_umem_release(obj->umem);
1630         kfree(obj);
1631         return 0;
1632 }
1633
1634 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
1635                                          size_t count, loff_t *pos)
1636 {
1637         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
1638         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1639         struct devx_async_data *event;
1640         int ret = 0;
1641         size_t eventsz;
1642
1643         spin_lock_irq(&ev_queue->lock);
1644
1645         while (list_empty(&ev_queue->event_list)) {
1646                 spin_unlock_irq(&ev_queue->lock);
1647
1648                 if (filp->f_flags & O_NONBLOCK)
1649                         return -EAGAIN;
1650
1651                 if (wait_event_interruptible(
1652                             ev_queue->poll_wait,
1653                             (!list_empty(&ev_queue->event_list) ||
1654                              ev_queue->is_destroyed))) {
1655                         return -ERESTARTSYS;
1656                 }
1657
1658                 if (list_empty(&ev_queue->event_list) &&
1659                     ev_queue->is_destroyed)
1660                         return -EIO;
1661
1662                 spin_lock_irq(&ev_queue->lock);
1663         }
1664
1665         event = list_entry(ev_queue->event_list.next,
1666                            struct devx_async_data, list);
1667         eventsz = event->cmd_out_len +
1668                         sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
1669
1670         if (eventsz > count) {
1671                 spin_unlock_irq(&ev_queue->lock);
1672                 return -ENOSPC;
1673         }
1674
1675         list_del(ev_queue->event_list.next);
1676         spin_unlock_irq(&ev_queue->lock);
1677
1678         if (copy_to_user(buf, &event->hdr, eventsz))
1679                 ret = -EFAULT;
1680         else
1681                 ret = eventsz;
1682
1683         atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
1684         kvfree(event);
1685         return ret;
1686 }
1687
1688 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp)
1689 {
1690         struct ib_uobject *uobj = filp->private_data;
1691         struct devx_async_cmd_event_file *comp_ev_file = container_of(
1692                 uobj, struct devx_async_cmd_event_file, uobj);
1693         struct devx_async_data *entry, *tmp;
1694
1695         spin_lock_irq(&comp_ev_file->ev_queue.lock);
1696         list_for_each_entry_safe(entry, tmp,
1697                                  &comp_ev_file->ev_queue.event_list, list)
1698                 kvfree(entry);
1699         spin_unlock_irq(&comp_ev_file->ev_queue.lock);
1700
1701         uverbs_close_fd(filp);
1702         return 0;
1703 }
1704
1705 static __poll_t devx_async_cmd_event_poll(struct file *filp,
1706                                               struct poll_table_struct *wait)
1707 {
1708         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
1709         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1710         __poll_t pollflags = 0;
1711
1712         poll_wait(filp, &ev_queue->poll_wait, wait);
1713
1714         spin_lock_irq(&ev_queue->lock);
1715         if (ev_queue->is_destroyed)
1716                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1717         else if (!list_empty(&ev_queue->event_list))
1718                 pollflags = EPOLLIN | EPOLLRDNORM;
1719         spin_unlock_irq(&ev_queue->lock);
1720
1721         return pollflags;
1722 }
1723
1724 static const struct file_operations devx_async_cmd_event_fops = {
1725         .owner   = THIS_MODULE,
1726         .read    = devx_async_cmd_event_read,
1727         .poll    = devx_async_cmd_event_poll,
1728         .release = devx_async_cmd_event_close,
1729         .llseek  = no_llseek,
1730 };
1731
1732 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
1733                                                    enum rdma_remove_reason why)
1734 {
1735         struct devx_async_cmd_event_file *comp_ev_file =
1736                 container_of(uobj, struct devx_async_cmd_event_file,
1737                              uobj);
1738         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1739
1740         spin_lock_irq(&ev_queue->lock);
1741         ev_queue->is_destroyed = 1;
1742         spin_unlock_irq(&ev_queue->lock);
1743
1744         if (why == RDMA_REMOVE_DRIVER_REMOVE)
1745                 wake_up_interruptible(&ev_queue->poll_wait);
1746
1747         mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
1748         return 0;
1749 };
1750
1751 DECLARE_UVERBS_NAMED_METHOD(
1752         MLX5_IB_METHOD_DEVX_UMEM_REG,
1753         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
1754                         MLX5_IB_OBJECT_DEVX_UMEM,
1755                         UVERBS_ACCESS_NEW,
1756                         UA_MANDATORY),
1757         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
1758                            UVERBS_ATTR_TYPE(u64),
1759                            UA_MANDATORY),
1760         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
1761                            UVERBS_ATTR_TYPE(u64),
1762                            UA_MANDATORY),
1763         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1764                              enum ib_access_flags),
1765         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
1766                             UVERBS_ATTR_TYPE(u32),
1767                             UA_MANDATORY));
1768
1769 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1770         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
1771         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
1772                         MLX5_IB_OBJECT_DEVX_UMEM,
1773                         UVERBS_ACCESS_DESTROY,
1774                         UA_MANDATORY));
1775
1776 DECLARE_UVERBS_NAMED_METHOD(
1777         MLX5_IB_METHOD_DEVX_QUERY_EQN,
1778         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
1779                            UVERBS_ATTR_TYPE(u32),
1780                            UA_MANDATORY),
1781         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1782                             UVERBS_ATTR_TYPE(u32),
1783                             UA_MANDATORY));
1784
1785 DECLARE_UVERBS_NAMED_METHOD(
1786         MLX5_IB_METHOD_DEVX_QUERY_UAR,
1787         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
1788                            UVERBS_ATTR_TYPE(u32),
1789                            UA_MANDATORY),
1790         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1791                             UVERBS_ATTR_TYPE(u32),
1792                             UA_MANDATORY));
1793
1794 DECLARE_UVERBS_NAMED_METHOD(
1795         MLX5_IB_METHOD_DEVX_OTHER,
1796         UVERBS_ATTR_PTR_IN(
1797                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
1798                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1799                 UA_MANDATORY,
1800                 UA_ALLOC_AND_COPY),
1801         UVERBS_ATTR_PTR_OUT(
1802                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
1803                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1804                 UA_MANDATORY));
1805
1806 DECLARE_UVERBS_NAMED_METHOD(
1807         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
1808         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
1809                         MLX5_IB_OBJECT_DEVX_OBJ,
1810                         UVERBS_ACCESS_NEW,
1811                         UA_MANDATORY),
1812         UVERBS_ATTR_PTR_IN(
1813                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
1814                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1815                 UA_MANDATORY,
1816                 UA_ALLOC_AND_COPY),
1817         UVERBS_ATTR_PTR_OUT(
1818                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1819                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1820                 UA_MANDATORY));
1821
1822 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1823         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
1824         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
1825                         MLX5_IB_OBJECT_DEVX_OBJ,
1826                         UVERBS_ACCESS_DESTROY,
1827                         UA_MANDATORY));
1828
1829 DECLARE_UVERBS_NAMED_METHOD(
1830         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
1831         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
1832                         UVERBS_IDR_ANY_OBJECT,
1833                         UVERBS_ACCESS_WRITE,
1834                         UA_MANDATORY),
1835         UVERBS_ATTR_PTR_IN(
1836                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
1837                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1838                 UA_MANDATORY,
1839                 UA_ALLOC_AND_COPY),
1840         UVERBS_ATTR_PTR_OUT(
1841                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1842                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1843                 UA_MANDATORY));
1844
1845 DECLARE_UVERBS_NAMED_METHOD(
1846         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
1847         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
1848                         UVERBS_IDR_ANY_OBJECT,
1849                         UVERBS_ACCESS_READ,
1850                         UA_MANDATORY),
1851         UVERBS_ATTR_PTR_IN(
1852                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
1853                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1854                 UA_MANDATORY,
1855                 UA_ALLOC_AND_COPY),
1856         UVERBS_ATTR_PTR_OUT(
1857                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1858                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1859                 UA_MANDATORY));
1860
1861 DECLARE_UVERBS_NAMED_METHOD(
1862         MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
1863         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
1864                         UVERBS_IDR_ANY_OBJECT,
1865                         UVERBS_ACCESS_READ,
1866                         UA_MANDATORY),
1867         UVERBS_ATTR_PTR_IN(
1868                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
1869                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1870                 UA_MANDATORY,
1871                 UA_ALLOC_AND_COPY),
1872         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
1873                 u16, UA_MANDATORY),
1874         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
1875                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1876                 UVERBS_ACCESS_READ,
1877                 UA_MANDATORY),
1878         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
1879                 UVERBS_ATTR_TYPE(u64),
1880                 UA_MANDATORY));
1881
1882 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
1883                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
1884                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
1885                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN));
1886
1887 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
1888                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
1889                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
1890                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
1891                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
1892                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
1893                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
1894
1895 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
1896                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
1897                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
1898                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
1899
1900
1901 DECLARE_UVERBS_NAMED_METHOD(
1902         MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
1903         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
1904                         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1905                         UVERBS_ACCESS_NEW,
1906                         UA_MANDATORY));
1907
1908 DECLARE_UVERBS_NAMED_OBJECT(
1909         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1910         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
1911                              devx_hot_unplug_async_cmd_event_file,
1912                              &devx_async_cmd_event_fops, "[devx_async_cmd]",
1913                              O_RDONLY),
1914         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
1915
1916 static bool devx_is_supported(struct ib_device *device)
1917 {
1918         struct mlx5_ib_dev *dev = to_mdev(device);
1919
1920         return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
1921 }
1922
1923 const struct uapi_definition mlx5_ib_devx_defs[] = {
1924         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1925                 MLX5_IB_OBJECT_DEVX,
1926                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1927         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1928                 MLX5_IB_OBJECT_DEVX_OBJ,
1929                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1930         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1931                 MLX5_IB_OBJECT_DEVX_UMEM,
1932                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1933         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1934                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1935                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1936         {},
1937 };