drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/mlx5_user_ioctl_verbs.h>
  12 #include <rdma/ib_umem.h>
  13 #include <rdma/uverbs_std_types.h>
  14 #include <linux/mlx5/driver.h>
  15 #include <linux/mlx5/fs.h>
  16 #include "mlx5_ib.h"
  17
  18 #define UVERBS_MODULE_NAME mlx5_ib
  19 #include <rdma/uverbs_named_ioctl.h>
  20
  21 enum devx_obj_flags {
  22         DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  23         DEVX_OBJ_FLAGS_DCT = 1 << 1,
  24 };
  25
  26 struct devx_async_data {
  27         struct mlx5_ib_dev *mdev;
  28         struct list_head list;
  29         struct ib_uobject *fd_uobj;
  30         struct mlx5_async_work cb_work;
  31         u16 cmd_out_len;
  32         /* must be last field in this structure */
  33         struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  34 };
  35
  36 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
  37 struct devx_obj {
  38         struct mlx5_core_dev    *mdev;
  39         u64                     obj_id;
  40         u32                     dinlen; /* destroy inbox length */
  41         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
  42         u32                     flags;
  43         union {
  44                 struct mlx5_ib_devx_mr  devx_mr;
  45                 struct mlx5_core_dct    core_dct;
  46         };
  47 };
  48
  49 struct devx_umem {
  50         struct mlx5_core_dev            *mdev;
  51         struct ib_umem                  *umem;
  52         u32                             page_offset;
  53         int                             page_shift;
  54         int                             ncont;
  55         u32                             dinlen;
  56         u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
  57 };
  58
  59 struct devx_umem_reg_cmd {
  60         void                            *in;
  61         u32                             inlen;
  62         u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
  63 };
  64
  65 static struct mlx5_ib_ucontext *
  66 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
  67 {
  68         return to_mucontext(ib_uverbs_get_ucontext(attrs));
  69 }
  70
  71 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
  72 {
  73         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
  74         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
  75         void *uctx;
  76         int err;
  77         u16 uid;
  78         u32 cap = 0;
  79
  80         /* 0 means not supported */
  81         if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
  82                 return -EINVAL;
  83
  84         uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
  85         if (is_user && capable(CAP_NET_RAW) &&
  86             (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
  87                 cap |= MLX5_UCTX_CAP_RAW_TX;
  88         if (is_user && capable(CAP_SYS_RAWIO) &&
  89             (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
  90              MLX5_UCTX_CAP_INTERNAL_DEV_RES))
  91                 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
  92
  93         MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
  94         MLX5_SET(uctx, uctx, cap, cap);
  95
  96         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
  97         if (err)
  98                 return err;
  99
 100         uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 101         return uid;
 102 }
 103
 104 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 105 {
 106         u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
 107         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 108
 109         MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 110         MLX5_SET(destroy_uctx_in, in, uid, uid);
 111
 112         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 113 }
 114
 115 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
 116 {
 117         struct devx_obj *devx_obj = obj;
 118         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 119
 120         switch (opcode) {
 121         case MLX5_CMD_OP_DESTROY_TIR:
 122                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 123                 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
 124                                     obj_id);
 125                 return true;
 126
 127         case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 128                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 129                 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
 130                                     table_id);
 131                 return true;
 132         default:
 133                 return false;
 134         }
 135 }
 136
 137 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id)
 138 {
 139         struct devx_obj *devx_obj = obj;
 140         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 141
 142         if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
 143                 *counter_id = MLX5_GET(dealloc_flow_counter_in,
 144                                        devx_obj->dinbox,
 145                                        flow_counter_id);
 146                 return true;
 147         }
 148
 149         return false;
 150 }
 151
 152 /*
 153  * As the obj_id in the firmware is not globally unique the object type
 154  * must be considered upon checking for a valid object id.
 155  * For that the opcode of the creator command is encoded as part of the obj_id.
 156  */
 157 static u64 get_enc_obj_id(u16 opcode, u32 obj_id)
 158 {
 159         return ((u64)opcode << 32) | obj_id;
 160 }
 161
 162 static u64 devx_get_obj_id(const void *in)
 163 {
 164         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 165         u64 obj_id;
 166
 167         switch (opcode) {
 168         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 169         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 170                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT,
 171                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 172                                                  obj_id));
 173                 break;
 174         case MLX5_CMD_OP_QUERY_MKEY:
 175                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 176                                         MLX5_GET(query_mkey_in, in,
 177                                                  mkey_index));
 178                 break;
 179         case MLX5_CMD_OP_QUERY_CQ:
 180                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 181                                         MLX5_GET(query_cq_in, in, cqn));
 182                 break;
 183         case MLX5_CMD_OP_MODIFY_CQ:
 184                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 185                                         MLX5_GET(modify_cq_in, in, cqn));
 186                 break;
 187         case MLX5_CMD_OP_QUERY_SQ:
 188                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 189                                         MLX5_GET(query_sq_in, in, sqn));
 190                 break;
 191         case MLX5_CMD_OP_MODIFY_SQ:
 192                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 193                                         MLX5_GET(modify_sq_in, in, sqn));
 194                 break;
 195         case MLX5_CMD_OP_QUERY_RQ:
 196                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 197                                         MLX5_GET(query_rq_in, in, rqn));
 198                 break;
 199         case MLX5_CMD_OP_MODIFY_RQ:
 200                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 201                                         MLX5_GET(modify_rq_in, in, rqn));
 202                 break;
 203         case MLX5_CMD_OP_QUERY_RMP:
 204                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 205                                         MLX5_GET(query_rmp_in, in, rmpn));
 206                 break;
 207         case MLX5_CMD_OP_MODIFY_RMP:
 208                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 209                                         MLX5_GET(modify_rmp_in, in, rmpn));
 210                 break;
 211         case MLX5_CMD_OP_QUERY_RQT:
 212                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 213                                         MLX5_GET(query_rqt_in, in, rqtn));
 214                 break;
 215         case MLX5_CMD_OP_MODIFY_RQT:
 216                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 217                                         MLX5_GET(modify_rqt_in, in, rqtn));
 218                 break;
 219         case MLX5_CMD_OP_QUERY_TIR:
 220                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 221                                         MLX5_GET(query_tir_in, in, tirn));
 222                 break;
 223         case MLX5_CMD_OP_MODIFY_TIR:
 224                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 225                                         MLX5_GET(modify_tir_in, in, tirn));
 226                 break;
 227         case MLX5_CMD_OP_QUERY_TIS:
 228                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 229                                         MLX5_GET(query_tis_in, in, tisn));
 230                 break;
 231         case MLX5_CMD_OP_MODIFY_TIS:
 232                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 233                                         MLX5_GET(modify_tis_in, in, tisn));
 234                 break;
 235         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 236                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 237                                         MLX5_GET(query_flow_table_in, in,
 238                                                  table_id));
 239                 break;
 240         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 241                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 242                                         MLX5_GET(modify_flow_table_in, in,
 243                                                  table_id));
 244                 break;
 245         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 246                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 247                                         MLX5_GET(query_flow_group_in, in,
 248                                                  group_id));
 249                 break;
 250         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 251                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 252                                         MLX5_GET(query_fte_in, in,
 253                                                  flow_index));
 254                 break;
 255         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 256                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 257                                         MLX5_GET(set_fte_in, in, flow_index));
 258                 break;
 259         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 260                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 261                                         MLX5_GET(query_q_counter_in, in,
 262                                                  counter_set_id));
 263                 break;
 264         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 265                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 266                                         MLX5_GET(query_flow_counter_in, in,
 267                                                  flow_counter_id));
 268                 break;
 269         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 270                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 271                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 272                                                  obj_id));
 273                 break;
 274         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 275                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 276                                         MLX5_GET(query_scheduling_element_in,
 277                                                  in, scheduling_element_id));
 278                 break;
 279         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 280                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 281                                         MLX5_GET(modify_scheduling_element_in,
 282                                                  in, scheduling_element_id));
 283                 break;
 284         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 285                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 286                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 287                                                  vxlan_udp_port));
 288                 break;
 289         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 290                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 291                                         MLX5_GET(query_l2_table_entry_in, in,
 292                                                  table_index));
 293                 break;
 294         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 295                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 296                                         MLX5_GET(set_l2_table_entry_in, in,
 297                                                  table_index));
 298                 break;
 299         case MLX5_CMD_OP_QUERY_QP:
 300                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 301                                         MLX5_GET(query_qp_in, in, qpn));
 302                 break;
 303         case MLX5_CMD_OP_RST2INIT_QP:
 304                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 305                                         MLX5_GET(rst2init_qp_in, in, qpn));
 306                 break;
 307         case MLX5_CMD_OP_INIT2RTR_QP:
 308                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 309                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 310                 break;
 311         case MLX5_CMD_OP_RTR2RTS_QP:
 312                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 313                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 314                 break;
 315         case MLX5_CMD_OP_RTS2RTS_QP:
 316                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 317                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 318                 break;
 319         case MLX5_CMD_OP_SQERR2RTS_QP:
 320                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 321                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 322                 break;
 323         case MLX5_CMD_OP_2ERR_QP:
 324                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 325                                         MLX5_GET(qp_2err_in, in, qpn));
 326                 break;
 327         case MLX5_CMD_OP_2RST_QP:
 328                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 329                                         MLX5_GET(qp_2rst_in, in, qpn));
 330                 break;
 331         case MLX5_CMD_OP_QUERY_DCT:
 332                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 333                                         MLX5_GET(query_dct_in, in, dctn));
 334                 break;
 335         case MLX5_CMD_OP_QUERY_XRQ:
 336         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 337         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 338                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 339                                         MLX5_GET(query_xrq_in, in, xrqn));
 340                 break;
 341         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 342                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 343                                         MLX5_GET(query_xrc_srq_in, in,
 344                                                  xrc_srqn));
 345                 break;
 346         case MLX5_CMD_OP_ARM_XRC_SRQ:
 347                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 348                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 349                 break;
 350         case MLX5_CMD_OP_QUERY_SRQ:
 351                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 352                                         MLX5_GET(query_srq_in, in, srqn));
 353                 break;
 354         case MLX5_CMD_OP_ARM_RQ:
 355                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 356                                         MLX5_GET(arm_rq_in, in, srq_number));
 357                 break;
 358         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 359                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 360                                         MLX5_GET(drain_dct_in, in, dctn));
 361                 break;
 362         case MLX5_CMD_OP_ARM_XRQ:
 363         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 364                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 365                                         MLX5_GET(arm_xrq_in, in, xrqn));
 366                 break;
 367         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 368                 obj_id = get_enc_obj_id
 369                                 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 370                                  MLX5_GET(query_packet_reformat_context_in,
 371                                           in, packet_reformat_id));
 372                 break;
 373         default:
 374                 obj_id = 0;
 375         }
 376
 377         return obj_id;
 378 }
 379
 380 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 381                                  struct ib_uobject *uobj, const void *in)
 382 {
 383         struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 384         u64 obj_id = devx_get_obj_id(in);
 385
 386         if (!obj_id)
 387                 return false;
 388
 389         switch (uobj_get_object_id(uobj)) {
 390         case UVERBS_OBJECT_CQ:
 391                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 392                                       to_mcq(uobj->object)->mcq.cqn) ==
 393                                       obj_id;
 394
 395         case UVERBS_OBJECT_SRQ:
 396         {
 397                 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 398                 u16 opcode;
 399
 400                 switch (srq->common.res) {
 401                 case MLX5_RES_XSRQ:
 402                         opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 403                         break;
 404                 case MLX5_RES_XRQ:
 405                         opcode = MLX5_CMD_OP_CREATE_XRQ;
 406                         break;
 407                 default:
 408                         if (!dev->mdev->issi)
 409                                 opcode = MLX5_CMD_OP_CREATE_SRQ;
 410                         else
 411                                 opcode = MLX5_CMD_OP_CREATE_RMP;
 412                 }
 413
 414                 return get_enc_obj_id(opcode,
 415                                       to_msrq(uobj->object)->msrq.srqn) ==
 416                                       obj_id;
 417         }
 418
 419         case UVERBS_OBJECT_QP:
 420         {
 421                 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 422                 enum ib_qp_type qp_type = qp->ibqp.qp_type;
 423
 424                 if (qp_type == IB_QPT_RAW_PACKET ||
 425                     (qp->flags & MLX5_IB_QP_UNDERLAY)) {
 426                         struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 427                                                          &qp->raw_packet_qp;
 428                         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 429                         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 430
 431                         return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 432                                                rq->base.mqp.qpn) == obj_id ||
 433                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 434                                                sq->base.mqp.qpn) == obj_id ||
 435                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 436                                                rq->tirn) == obj_id ||
 437                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 438                                                sq->tisn) == obj_id);
 439                 }
 440
 441                 if (qp_type == MLX5_IB_QPT_DCT)
 442                         return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 443                                               qp->dct.mdct.mqp.qpn) == obj_id;
 444
 445                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 446                                       qp->ibqp.qp_num) == obj_id;
 447         }
 448
 449         case UVERBS_OBJECT_WQ:
 450                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 451                                       to_mrwq(uobj->object)->core_qp.qpn) ==
 452                                       obj_id;
 453
 454         case UVERBS_OBJECT_RWQ_IND_TBL:
 455                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 456                                       to_mrwq_ind_table(uobj->object)->rqtn) ==
 457                                       obj_id;
 458
 459         case MLX5_IB_OBJECT_DEVX_OBJ:
 460                 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
 461
 462         default:
 463                 return false;
 464         }
 465 }
 466
 467 static void devx_set_umem_valid(const void *in)
 468 {
 469         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 470
 471         switch (opcode) {
 472         case MLX5_CMD_OP_CREATE_MKEY:
 473                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 474                 break;
 475         case MLX5_CMD_OP_CREATE_CQ:
 476         {
 477                 void *cqc;
 478
 479                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 480                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 481                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 482                 break;
 483         }
 484         case MLX5_CMD_OP_CREATE_QP:
 485         {
 486                 void *qpc;
 487
 488                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 489                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 490                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 491                 break;
 492         }
 493
 494         case MLX5_CMD_OP_CREATE_RQ:
 495         {
 496                 void *rqc, *wq;
 497
 498                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 499                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 500                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 501                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 502                 break;
 503         }
 504
 505         case MLX5_CMD_OP_CREATE_SQ:
 506         {
 507                 void *sqc, *wq;
 508
 509                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 510                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 511                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 512                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 513                 break;
 514         }
 515
 516         case MLX5_CMD_OP_MODIFY_CQ:
 517                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 518                 break;
 519
 520         case MLX5_CMD_OP_CREATE_RMP:
 521         {
 522                 void *rmpc, *wq;
 523
 524                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 525                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 526                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 527                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 528                 break;
 529         }
 530
 531         case MLX5_CMD_OP_CREATE_XRQ:
 532         {
 533                 void *xrqc, *wq;
 534
 535                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 536                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 537                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 538                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 539                 break;
 540         }
 541
 542         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 543         {
 544                 void *xrc_srqc;
 545
 546                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 547                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 548                                         xrc_srq_context_entry);
 549                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 550                 break;
 551         }
 552
 553         default:
 554                 return;
 555         }
 556 }
 557
 558 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 559 {
 560         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 561
 562         switch (*opcode) {
 563         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 564         case MLX5_CMD_OP_CREATE_MKEY:
 565         case MLX5_CMD_OP_CREATE_CQ:
 566         case MLX5_CMD_OP_ALLOC_PD:
 567         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 568         case MLX5_CMD_OP_CREATE_RMP:
 569         case MLX5_CMD_OP_CREATE_SQ:
 570         case MLX5_CMD_OP_CREATE_RQ:
 571         case MLX5_CMD_OP_CREATE_RQT:
 572         case MLX5_CMD_OP_CREATE_TIR:
 573         case MLX5_CMD_OP_CREATE_TIS:
 574         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 575         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 576         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 577         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 578         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 579         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 580         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 581         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 582         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 583         case MLX5_CMD_OP_CREATE_QP:
 584         case MLX5_CMD_OP_CREATE_SRQ:
 585         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 586         case MLX5_CMD_OP_CREATE_DCT:
 587         case MLX5_CMD_OP_CREATE_XRQ:
 588         case MLX5_CMD_OP_ATTACH_TO_MCG:
 589         case MLX5_CMD_OP_ALLOC_XRCD:
 590                 return true;
 591         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 592         {
 593                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 594                 if (op_mod == 0)
 595                         return true;
 596                 return false;
 597         }
 598         default:
 599                 return false;
 600         }
 601 }
 602
 603 static bool devx_is_obj_modify_cmd(const void *in)
 604 {
 605         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 606
 607         switch (opcode) {
 608         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 609         case MLX5_CMD_OP_MODIFY_CQ:
 610         case MLX5_CMD_OP_MODIFY_RMP:
 611         case MLX5_CMD_OP_MODIFY_SQ:
 612         case MLX5_CMD_OP_MODIFY_RQ:
 613         case MLX5_CMD_OP_MODIFY_RQT:
 614         case MLX5_CMD_OP_MODIFY_TIR:
 615         case MLX5_CMD_OP_MODIFY_TIS:
 616         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 617         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 618         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 619         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 620         case MLX5_CMD_OP_RST2INIT_QP:
 621         case MLX5_CMD_OP_INIT2RTR_QP:
 622         case MLX5_CMD_OP_RTR2RTS_QP:
 623         case MLX5_CMD_OP_RTS2RTS_QP:
 624         case MLX5_CMD_OP_SQERR2RTS_QP:
 625         case MLX5_CMD_OP_2ERR_QP:
 626         case MLX5_CMD_OP_2RST_QP:
 627         case MLX5_CMD_OP_ARM_XRC_SRQ:
 628         case MLX5_CMD_OP_ARM_RQ:
 629         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 630         case MLX5_CMD_OP_ARM_XRQ:
 631         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 632                 return true;
 633         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 634         {
 635                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 636
 637                 if (op_mod == 1)
 638                         return true;
 639                 return false;
 640         }
 641         default:
 642                 return false;
 643         }
 644 }
 645
 646 static bool devx_is_obj_query_cmd(const void *in)
 647 {
 648         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 649
 650         switch (opcode) {
 651         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 652         case MLX5_CMD_OP_QUERY_MKEY:
 653         case MLX5_CMD_OP_QUERY_CQ:
 654         case MLX5_CMD_OP_QUERY_RMP:
 655         case MLX5_CMD_OP_QUERY_SQ:
 656         case MLX5_CMD_OP_QUERY_RQ:
 657         case MLX5_CMD_OP_QUERY_RQT:
 658         case MLX5_CMD_OP_QUERY_TIR:
 659         case MLX5_CMD_OP_QUERY_TIS:
 660         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 661         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 662         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 663         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 664         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 665         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 666         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 667         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 668         case MLX5_CMD_OP_QUERY_QP:
 669         case MLX5_CMD_OP_QUERY_SRQ:
 670         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 671         case MLX5_CMD_OP_QUERY_DCT:
 672         case MLX5_CMD_OP_QUERY_XRQ:
 673         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 674         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 675         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 676                 return true;
 677         default:
 678                 return false;
 679         }
 680 }
 681
 682 static bool devx_is_whitelist_cmd(void *in)
 683 {
 684         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 685
 686         switch (opcode) {
 687         case MLX5_CMD_OP_QUERY_HCA_CAP:
 688         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 689         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 690                 return true;
 691         default:
 692                 return false;
 693         }
 694 }
 695
 696 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 697 {
 698         if (devx_is_whitelist_cmd(cmd_in)) {
 699                 struct mlx5_ib_dev *dev;
 700
 701                 if (c->devx_uid)
 702                         return c->devx_uid;
 703
 704                 dev = to_mdev(c->ibucontext.device);
 705                 if (dev->devx_whitelist_uid)
 706                         return dev->devx_whitelist_uid;
 707
 708                 return -EOPNOTSUPP;
 709         }
 710
 711         if (!c->devx_uid)
 712                 return -EINVAL;
 713
 714         return c->devx_uid;
 715 }
 716 static bool devx_is_general_cmd(void *in)
 717 {
 718         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 719
 720         if (opcode >= MLX5_CMD_OP_GENERAL_START &&
 721             opcode < MLX5_CMD_OP_GENERAL_END)
 722                 return true;
 723
 724         switch (opcode) {
 725         case MLX5_CMD_OP_QUERY_HCA_CAP:
 726         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 727         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 728         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 729         case MLX5_CMD_OP_QUERY_ADAPTER:
 730         case MLX5_CMD_OP_QUERY_ISSI:
 731         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 732         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 733         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 734         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 735         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 736         case MLX5_CMD_OP_NOP:
 737         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 738         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 739         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 740                 return true;
 741         default:
 742                 return false;
 743         }
 744 }
 745
 746 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 747         struct uverbs_attr_bundle *attrs)
 748 {
 749         struct mlx5_ib_ucontext *c;
 750         struct mlx5_ib_dev *dev;
 751         int user_vector;
 752         int dev_eqn;
 753         unsigned int irqn;
 754         int err;
 755
 756         if (uverbs_copy_from(&user_vector, attrs,
 757                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 758                 return -EFAULT;
 759
 760         c = devx_ufile2uctx(attrs);
 761         if (IS_ERR(c))
 762                 return PTR_ERR(c);
 763         dev = to_mdev(c->ibucontext.device);
 764
 765         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
 766         if (err < 0)
 767                 return err;
 768
 769         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 770                            &dev_eqn, sizeof(dev_eqn)))
 771                 return -EFAULT;
 772
 773         return 0;
 774 }
 775
 776 /*
 777  *Security note:
 778  * The hardware protection mechanism works like this: Each device object that
 779  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
 780  * the device specification manual) upon its creation. Then upon doorbell,
 781  * hardware fetches the object context for which the doorbell was rang, and
 782  * validates that the UAR through which the DB was rang matches the UAR ID
 783  * of the object.
 784  * If no match the doorbell is silently ignored by the hardware. Of course,
 785  * the user cannot ring a doorbell on a UAR that was not mapped to it.
 786  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
 787  * mailboxes (except tagging them with UID), we expose to the user its UAR
 788  * ID, so it can embed it in these objects in the expected specification
 789  * format. So the only thing the user can do is hurt itself by creating a
 790  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
 791  * may ring a doorbell on its objects.
 792  * The consequence of that will be that another user can schedule a QP/SQ
 793  * of the buggy user for execution (just insert it to the hardware schedule
 794  * queue or arm its CQ for event generation), no further harm is expected.
 795  */
 796 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
 797         struct uverbs_attr_bundle *attrs)
 798 {
 799         struct mlx5_ib_ucontext *c;
 800         struct mlx5_ib_dev *dev;
 801         u32 user_idx;
 802         s32 dev_idx;
 803
 804         c = devx_ufile2uctx(attrs);
 805         if (IS_ERR(c))
 806                 return PTR_ERR(c);
 807         dev = to_mdev(c->ibucontext.device);
 808
 809         if (uverbs_copy_from(&user_idx, attrs,
 810                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
 811                 return -EFAULT;
 812
 813         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
 814         if (dev_idx < 0)
 815                 return dev_idx;
 816
 817         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
 818                            &dev_idx, sizeof(dev_idx)))
 819                 return -EFAULT;
 820
 821         return 0;
 822 }
 823
 824 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
 825         struct uverbs_attr_bundle *attrs)
 826 {
 827         struct mlx5_ib_ucontext *c;
 828         struct mlx5_ib_dev *dev;
 829         void *cmd_in = uverbs_attr_get_alloced_ptr(
 830                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
 831         int cmd_out_len = uverbs_attr_get_len(attrs,
 832                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
 833         void *cmd_out;
 834         int err;
 835         int uid;
 836
 837         c = devx_ufile2uctx(attrs);
 838         if (IS_ERR(c))
 839                 return PTR_ERR(c);
 840         dev = to_mdev(c->ibucontext.device);
 841
 842         uid = devx_get_uid(c, cmd_in);
 843         if (uid < 0)
 844                 return uid;
 845
 846         /* Only white list of some general HCA commands are allowed for this method. */
 847         if (!devx_is_general_cmd(cmd_in))
 848                 return -EINVAL;
 849
 850         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
 851         if (IS_ERR(cmd_out))
 852                 return PTR_ERR(cmd_out);
 853
 854         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
 855         err = mlx5_cmd_exec(dev->mdev, cmd_in,
 856                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
 857                             cmd_out, cmd_out_len);
 858         if (err)
 859                 return err;
 860
 861         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
 862                               cmd_out_len);
 863 }
 864
 865 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
 866                                        u32 *dinlen,
 867                                        u32 *obj_id)
 868 {
 869         u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
 870         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
 871
 872         *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 873         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
 874
 875         MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
 876         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
 877
 878         switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
 879         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 880                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
 881                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
 882                 break;
 883
 884         case MLX5_CMD_OP_CREATE_UMEM:
 885                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 886                          MLX5_CMD_OP_DESTROY_UMEM);
 887                 break;
 888         case MLX5_CMD_OP_CREATE_MKEY:
 889                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
 890                 break;
 891         case MLX5_CMD_OP_CREATE_CQ:
 892                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
 893                 break;
 894         case MLX5_CMD_OP_ALLOC_PD:
 895                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
 896                 break;
 897         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 898                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 899                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 900                 break;
 901         case MLX5_CMD_OP_CREATE_RMP:
 902                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
 903                 break;
 904         case MLX5_CMD_OP_CREATE_SQ:
 905                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
 906                 break;
 907         case MLX5_CMD_OP_CREATE_RQ:
 908                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
 909                 break;
 910         case MLX5_CMD_OP_CREATE_RQT:
 911                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
 912                 break;
 913         case MLX5_CMD_OP_CREATE_TIR:
 914                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
 915                 break;
 916         case MLX5_CMD_OP_CREATE_TIS:
 917                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
 918                 break;
 919         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 920                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 921                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 922                 break;
 923         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 924                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
 925                 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
 926                 MLX5_SET(destroy_flow_table_in, din, other_vport,
 927                          MLX5_GET(create_flow_table_in,  in, other_vport));
 928                 MLX5_SET(destroy_flow_table_in, din, vport_number,
 929                          MLX5_GET(create_flow_table_in,  in, vport_number));
 930                 MLX5_SET(destroy_flow_table_in, din, table_type,
 931                          MLX5_GET(create_flow_table_in,  in, table_type));
 932                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
 933                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 934                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
 935                 break;
 936         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 937                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
 938                 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
 939                 MLX5_SET(destroy_flow_group_in, din, other_vport,
 940                          MLX5_GET(create_flow_group_in, in, other_vport));
 941                 MLX5_SET(destroy_flow_group_in, din, vport_number,
 942                          MLX5_GET(create_flow_group_in, in, vport_number));
 943                 MLX5_SET(destroy_flow_group_in, din, table_type,
 944                          MLX5_GET(create_flow_group_in, in, table_type));
 945                 MLX5_SET(destroy_flow_group_in, din, table_id,
 946                          MLX5_GET(create_flow_group_in, in, table_id));
 947                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
 948                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 949                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
 950                 break;
 951         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 952                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
 953                 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
 954                 MLX5_SET(delete_fte_in, din, other_vport,
 955                          MLX5_GET(set_fte_in,  in, other_vport));
 956                 MLX5_SET(delete_fte_in, din, vport_number,
 957                          MLX5_GET(set_fte_in, in, vport_number));
 958                 MLX5_SET(delete_fte_in, din, table_type,
 959                          MLX5_GET(set_fte_in, in, table_type));
 960                 MLX5_SET(delete_fte_in, din, table_id,
 961                          MLX5_GET(set_fte_in, in, table_id));
 962                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
 963                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 964                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
 965                 break;
 966         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 967                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 968                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
 969                 break;
 970         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 971                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 972                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
 973                 break;
 974         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 975                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 976                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
 977                 break;
 978         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 979                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
 980                 *obj_id = MLX5_GET(create_scheduling_element_out, out,
 981                                    scheduling_element_id);
 982                 MLX5_SET(destroy_scheduling_element_in, din,
 983                          scheduling_hierarchy,
 984                          MLX5_GET(create_scheduling_element_in, in,
 985                                   scheduling_hierarchy));
 986                 MLX5_SET(destroy_scheduling_element_in, din,
 987                          scheduling_element_id, *obj_id);
 988                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 989                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
 990                 break;
 991         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 992                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
 993                 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
 994                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
 995                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 996                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
 997                 break;
 998         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 999                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1000                 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1001                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1002                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1003                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1004                 break;
1005         case MLX5_CMD_OP_CREATE_QP:
1006                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1007                 break;
1008         case MLX5_CMD_OP_CREATE_SRQ:
1009                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1010                 break;
1011         case MLX5_CMD_OP_CREATE_XRC_SRQ:
1012                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1013                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
1014                 break;
1015         case MLX5_CMD_OP_CREATE_DCT:
1016                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1017                 break;
1018         case MLX5_CMD_OP_CREATE_XRQ:
1019                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1020                 break;
1021         case MLX5_CMD_OP_ATTACH_TO_MCG:
1022                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1023                 MLX5_SET(detach_from_mcg_in, din, qpn,
1024                          MLX5_GET(attach_to_mcg_in, in, qpn));
1025                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1026                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1027                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1028                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1029                 break;
1030         case MLX5_CMD_OP_ALLOC_XRCD:
1031                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1032                 break;
1033         default:
1034                 /* The entry must match to one of the devx_is_obj_create_cmd */
1035                 WARN_ON(true);
1036                 break;
1037         }
1038 }
1039
1040 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1041                                      struct mlx5_ib_dev *dev,
1042                                      void *in, void *out)
1043 {
1044         struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
1045         struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1046         unsigned long flags;
1047         struct mlx5_core_mkey *mkey;
1048         void *mkc;
1049         u8 key;
1050         int err;
1051
1052         mkey = &devx_mr->mmkey;
1053         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1054         key = MLX5_GET(mkc, mkc, mkey_7_0);
1055         mkey->key = mlx5_idx_to_mkey(
1056                         MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1057         mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1058         mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1059         mkey->size = MLX5_GET64(mkc, mkc, len);
1060         mkey->pd = MLX5_GET(mkc, mkc, pd);
1061         devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1062
1063         write_lock_irqsave(&table->lock, flags);
1064         err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key),
1065                                 mkey);
1066         write_unlock_irqrestore(&table->lock, flags);
1067         return err;
1068 }
1069
1070 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1071                                    struct devx_obj *obj,
1072                                    void *in, int in_len)
1073 {
1074         int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1075                         MLX5_FLD_SZ_BYTES(create_mkey_in,
1076                         memory_key_mkey_entry);
1077         void *mkc;
1078         u8 access_mode;
1079
1080         if (in_len < min_len)
1081                 return -EINVAL;
1082
1083         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1084
1085         access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1086         access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1087
1088         if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1089                 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1090                 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1091                         obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1092                 return 0;
1093         }
1094
1095         MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1096         return 0;
1097 }
1098
1099 static void devx_free_indirect_mkey(struct rcu_head *rcu)
1100 {
1101         kfree(container_of(rcu, struct devx_obj, devx_mr.rcu));
1102 }
1103
1104 /* This function to delete from the radix tree needs to be called before
1105  * destroying the underlying mkey. Otherwise a race might occur in case that
1106  * other thread will get the same mkey before this one will be deleted,
1107  * in that case it will fail via inserting to the tree its own data.
1108  *
1109  * Note:
1110  * An error in the destroy is not expected unless there is some other indirect
1111  * mkey which points to this one. In a kernel cleanup flow it will be just
1112  * destroyed in the iterative destruction call. In a user flow, in case
1113  * the application didn't close in the expected order it's its own problem,
1114  * the mkey won't be part of the tree, in both cases the kernel is safe.
1115  */
1116 static void devx_cleanup_mkey(struct devx_obj *obj)
1117 {
1118         struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table;
1119         unsigned long flags;
1120
1121         write_lock_irqsave(&table->lock, flags);
1122         radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key));
1123         write_unlock_irqrestore(&table->lock, flags);
1124 }
1125
1126 static int devx_obj_cleanup(struct ib_uobject *uobject,
1127                             enum rdma_remove_reason why,
1128                             struct uverbs_attr_bundle *attrs)
1129 {
1130         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1131         struct devx_obj *obj = uobject->object;
1132         int ret;
1133
1134         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY)
1135                 devx_cleanup_mkey(obj);
1136
1137         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1138                 ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
1139         else
1140                 ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
1141                                     sizeof(out));
1142         if (ib_is_destroy_retryable(ret, why, uobject))
1143                 return ret;
1144
1145         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1146                 struct mlx5_ib_dev *dev =
1147                         mlx5_udata_to_mdev(&attrs->driver_udata);
1148
1149                 call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu,
1150                           devx_free_indirect_mkey);
1151                 return ret;
1152         }
1153
1154         kfree(obj);
1155         return ret;
1156 }
1157
1158 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1159         struct uverbs_attr_bundle *attrs)
1160 {
1161         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1162         int cmd_out_len =  uverbs_attr_get_len(attrs,
1163                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1164         int cmd_in_len = uverbs_attr_get_len(attrs,
1165                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1166         void *cmd_out;
1167         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1168                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1169         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1170                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1171         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1172         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1173         struct devx_obj *obj;
1174         int err;
1175         int uid;
1176         u32 obj_id;
1177         u16 opcode;
1178
1179         uid = devx_get_uid(c, cmd_in);
1180         if (uid < 0)
1181                 return uid;
1182
1183         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1184                 return -EINVAL;
1185
1186         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1187         if (IS_ERR(cmd_out))
1188                 return PTR_ERR(cmd_out);
1189
1190         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1191         if (!obj)
1192                 return -ENOMEM;
1193
1194         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1195         if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1196                 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1197                 if (err)
1198                         goto obj_free;
1199         } else {
1200                 devx_set_umem_valid(cmd_in);
1201         }
1202
1203         if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1204                 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1205                 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct,
1206                                            cmd_in, cmd_in_len,
1207                                            cmd_out, cmd_out_len);
1208         } else {
1209                 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1210                                     cmd_in_len,
1211                                     cmd_out, cmd_out_len);
1212         }
1213
1214         if (err)
1215                 goto obj_free;
1216
1217         uobj->object = obj;
1218         obj->mdev = dev->mdev;
1219         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1220                                    &obj_id);
1221         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1222
1223         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1224                 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1225                 if (err)
1226                         goto obj_destroy;
1227         }
1228
1229         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1230         if (err)
1231                 goto err_copy;
1232
1233         obj->obj_id = get_enc_obj_id(opcode, obj_id);
1234         return 0;
1235
1236 err_copy:
1237         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY)
1238                 devx_cleanup_mkey(obj);
1239 obj_destroy:
1240         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1241                 mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
1242         else
1243                 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
1244                               sizeof(out));
1245 obj_free:
1246         kfree(obj);
1247         return err;
1248 }
1249
1250 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1251         struct uverbs_attr_bundle *attrs)
1252 {
1253         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1254         int cmd_out_len = uverbs_attr_get_len(attrs,
1255                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1256         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1257                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1258         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1259                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1260         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1261         void *cmd_out;
1262         int err;
1263         int uid;
1264
1265         uid = devx_get_uid(c, cmd_in);
1266         if (uid < 0)
1267                 return uid;
1268
1269         if (!devx_is_obj_modify_cmd(cmd_in))
1270                 return -EINVAL;
1271
1272         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1273                 return -EINVAL;
1274
1275         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1276         if (IS_ERR(cmd_out))
1277                 return PTR_ERR(cmd_out);
1278
1279         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1280         devx_set_umem_valid(cmd_in);
1281
1282         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1283                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1284                             cmd_out, cmd_out_len);
1285         if (err)
1286                 return err;
1287
1288         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1289                               cmd_out, cmd_out_len);
1290 }
1291
1292 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1293         struct uverbs_attr_bundle *attrs)
1294 {
1295         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1296         int cmd_out_len = uverbs_attr_get_len(attrs,
1297                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1298         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1299                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1300         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1301                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1302         void *cmd_out;
1303         int err;
1304         int uid;
1305         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1306
1307         uid = devx_get_uid(c, cmd_in);
1308         if (uid < 0)
1309                 return uid;
1310
1311         if (!devx_is_obj_query_cmd(cmd_in))
1312                 return -EINVAL;
1313
1314         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1315                 return -EINVAL;
1316
1317         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1318         if (IS_ERR(cmd_out))
1319                 return PTR_ERR(cmd_out);
1320
1321         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1322         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1323                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1324                             cmd_out, cmd_out_len);
1325         if (err)
1326                 return err;
1327
1328         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1329                               cmd_out, cmd_out_len);
1330 }
1331
1332 struct devx_async_event_queue {
1333         spinlock_t              lock;
1334         wait_queue_head_t       poll_wait;
1335         struct list_head        event_list;
1336         atomic_t                bytes_in_use;
1337         u8                      is_destroyed:1;
1338 };
1339
1340 struct devx_async_cmd_event_file {
1341         struct ib_uobject               uobj;
1342         struct devx_async_event_queue   ev_queue;
1343         struct mlx5_async_ctx           async_ctx;
1344 };
1345
1346 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1347 {
1348         spin_lock_init(&ev_queue->lock);
1349         INIT_LIST_HEAD(&ev_queue->event_list);
1350         init_waitqueue_head(&ev_queue->poll_wait);
1351         atomic_set(&ev_queue->bytes_in_use, 0);
1352         ev_queue->is_destroyed = 0;
1353 }
1354
1355 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1356         struct uverbs_attr_bundle *attrs)
1357 {
1358         struct devx_async_cmd_event_file *ev_file;
1359
1360         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1361                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1362         struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1363
1364         ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1365                                uobj);
1366         devx_init_event_queue(&ev_file->ev_queue);
1367         mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1368         return 0;
1369 }
1370
1371 static void devx_query_callback(int status, struct mlx5_async_work *context)
1372 {
1373         struct devx_async_data *async_data =
1374                 container_of(context, struct devx_async_data, cb_work);
1375         struct ib_uobject *fd_uobj = async_data->fd_uobj;
1376         struct devx_async_cmd_event_file *ev_file;
1377         struct devx_async_event_queue *ev_queue;
1378         unsigned long flags;
1379
1380         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1381                                uobj);
1382         ev_queue = &ev_file->ev_queue;
1383
1384         spin_lock_irqsave(&ev_queue->lock, flags);
1385         list_add_tail(&async_data->list, &ev_queue->event_list);
1386         spin_unlock_irqrestore(&ev_queue->lock, flags);
1387
1388         wake_up_interruptible(&ev_queue->poll_wait);
1389         fput(fd_uobj->object);
1390 }
1391
1392 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1393
1394 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1395         struct uverbs_attr_bundle *attrs)
1396 {
1397         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1398                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1399         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1400                                 attrs,
1401                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1402         u16 cmd_out_len;
1403         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1404                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1405         struct ib_uobject *fd_uobj;
1406         int err;
1407         int uid;
1408         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1409         struct devx_async_cmd_event_file *ev_file;
1410         struct devx_async_data *async_data;
1411
1412         uid = devx_get_uid(c, cmd_in);
1413         if (uid < 0)
1414                 return uid;
1415
1416         if (!devx_is_obj_query_cmd(cmd_in))
1417                 return -EINVAL;
1418
1419         err = uverbs_get_const(&cmd_out_len, attrs,
1420                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1421         if (err)
1422                 return err;
1423
1424         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1425                 return -EINVAL;
1426
1427         fd_uobj = uverbs_attr_get_uobject(attrs,
1428                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1429         if (IS_ERR(fd_uobj))
1430                 return PTR_ERR(fd_uobj);
1431
1432         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1433                                uobj);
1434
1435         if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1436                         MAX_ASYNC_BYTES_IN_USE) {
1437                 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1438                 return -EAGAIN;
1439         }
1440
1441         async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1442                                           cmd_out_len), GFP_KERNEL);
1443         if (!async_data) {
1444                 err = -ENOMEM;
1445                 goto sub_bytes;
1446         }
1447
1448         err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1449                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1450         if (err)
1451                 goto free_async;
1452
1453         async_data->cmd_out_len = cmd_out_len;
1454         async_data->mdev = mdev;
1455         async_data->fd_uobj = fd_uobj;
1456
1457         get_file(fd_uobj->object);
1458         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1459         err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1460                     uverbs_attr_get_len(attrs,
1461                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1462                     async_data->hdr.out_data,
1463                     async_data->cmd_out_len,
1464                     devx_query_callback, &async_data->cb_work);
1465
1466         if (err)
1467                 goto cb_err;
1468
1469         return 0;
1470
1471 cb_err:
1472         fput(fd_uobj->object);
1473 free_async:
1474         kvfree(async_data);
1475 sub_bytes:
1476         atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1477         return err;
1478 }
1479
1480 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
1481                          struct uverbs_attr_bundle *attrs,
1482                          struct devx_umem *obj)
1483 {
1484         u64 addr;
1485         size_t size;
1486         u32 access;
1487         int npages;
1488         int err;
1489         u32 page_mask;
1490
1491         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
1492             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
1493                 return -EFAULT;
1494
1495         err = uverbs_get_flags32(&access, attrs,
1496                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1497                                  IB_ACCESS_LOCAL_WRITE |
1498                                  IB_ACCESS_REMOTE_WRITE |
1499                                  IB_ACCESS_REMOTE_READ);
1500         if (err)
1501                 return err;
1502
1503         err = ib_check_mr_access(access);
1504         if (err)
1505                 return err;
1506
1507         obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0);
1508         if (IS_ERR(obj->umem))
1509                 return PTR_ERR(obj->umem);
1510
1511         mlx5_ib_cont_pages(obj->umem, obj->umem->address,
1512                            MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
1513                            &obj->page_shift, &obj->ncont, NULL);
1514
1515         if (!npages) {
1516                 ib_umem_release(obj->umem);
1517                 return -EINVAL;
1518         }
1519
1520         page_mask = (1 << obj->page_shift) - 1;
1521         obj->page_offset = obj->umem->address & page_mask;
1522
1523         return 0;
1524 }
1525
1526 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
1527                                    struct devx_umem *obj,
1528                                    struct devx_umem_reg_cmd *cmd)
1529 {
1530         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
1531                     (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
1532         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
1533         return PTR_ERR_OR_ZERO(cmd->in);
1534 }
1535
1536 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
1537                                     struct devx_umem *obj,
1538                                     struct devx_umem_reg_cmd *cmd)
1539 {
1540         void *umem;
1541         __be64 *mtt;
1542
1543         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
1544         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
1545
1546         MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
1547         MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
1548         MLX5_SET(umem, umem, log_page_size, obj->page_shift -
1549                                             MLX5_ADAPTER_PAGE_SHIFT);
1550         MLX5_SET(umem, umem, page_offset, obj->page_offset);
1551         mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
1552                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
1553                              MLX5_IB_MTT_READ);
1554 }
1555
1556 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
1557         struct uverbs_attr_bundle *attrs)
1558 {
1559         struct devx_umem_reg_cmd cmd;
1560         struct devx_umem *obj;
1561         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1562                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
1563         u32 obj_id;
1564         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1565                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1566         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1567         int err;
1568
1569         if (!c->devx_uid)
1570                 return -EINVAL;
1571
1572         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
1573         if (!obj)
1574                 return -ENOMEM;
1575
1576         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
1577         if (err)
1578                 goto err_obj_free;
1579
1580         err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
1581         if (err)
1582                 goto err_umem_release;
1583
1584         devx_umem_reg_cmd_build(dev, obj, &cmd);
1585
1586         MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
1587         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
1588                             sizeof(cmd.out));
1589         if (err)
1590                 goto err_umem_release;
1591
1592         obj->mdev = dev->mdev;
1593         uobj->object = obj;
1594         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
1595         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
1596         if (err)
1597                 goto err_umem_destroy;
1598
1599         return 0;
1600
1601 err_umem_destroy:
1602         mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
1603 err_umem_release:
1604         ib_umem_release(obj->umem);
1605 err_obj_free:
1606         kfree(obj);
1607         return err;
1608 }
1609
1610 static int devx_umem_cleanup(struct ib_uobject *uobject,
1611                              enum rdma_remove_reason why,
1612                              struct uverbs_attr_bundle *attrs)
1613 {
1614         struct devx_umem *obj = uobject->object;
1615         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1616         int err;
1617
1618         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
1619         if (ib_is_destroy_retryable(err, why, uobject))
1620                 return err;
1621
1622         ib_umem_release(obj->umem);
1623         kfree(obj);
1624         return 0;
1625 }
1626
1627 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
1628                                          size_t count, loff_t *pos)
1629 {
1630         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
1631         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1632         struct devx_async_data *event;
1633         int ret = 0;
1634         size_t eventsz;
1635
1636         spin_lock_irq(&ev_queue->lock);
1637
1638         while (list_empty(&ev_queue->event_list)) {
1639                 spin_unlock_irq(&ev_queue->lock);
1640
1641                 if (filp->f_flags & O_NONBLOCK)
1642                         return -EAGAIN;
1643
1644                 if (wait_event_interruptible(
1645                             ev_queue->poll_wait,
1646                             (!list_empty(&ev_queue->event_list) ||
1647                              ev_queue->is_destroyed))) {
1648                         return -ERESTARTSYS;
1649                 }
1650
1651                 if (list_empty(&ev_queue->event_list) &&
1652                     ev_queue->is_destroyed)
1653                         return -EIO;
1654
1655                 spin_lock_irq(&ev_queue->lock);
1656         }
1657
1658         event = list_entry(ev_queue->event_list.next,
1659                            struct devx_async_data, list);
1660         eventsz = event->cmd_out_len +
1661                         sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
1662
1663         if (eventsz > count) {
1664                 spin_unlock_irq(&ev_queue->lock);
1665                 return -ENOSPC;
1666         }
1667
1668         list_del(ev_queue->event_list.next);
1669         spin_unlock_irq(&ev_queue->lock);
1670
1671         if (copy_to_user(buf, &event->hdr, eventsz))
1672                 ret = -EFAULT;
1673         else
1674                 ret = eventsz;
1675
1676         atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
1677         kvfree(event);
1678         return ret;
1679 }
1680
1681 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp)
1682 {
1683         struct ib_uobject *uobj = filp->private_data;
1684         struct devx_async_cmd_event_file *comp_ev_file = container_of(
1685                 uobj, struct devx_async_cmd_event_file, uobj);
1686         struct devx_async_data *entry, *tmp;
1687
1688         spin_lock_irq(&comp_ev_file->ev_queue.lock);
1689         list_for_each_entry_safe(entry, tmp,
1690                                  &comp_ev_file->ev_queue.event_list, list)
1691                 kvfree(entry);
1692         spin_unlock_irq(&comp_ev_file->ev_queue.lock);
1693
1694         uverbs_close_fd(filp);
1695         return 0;
1696 }
1697
1698 static __poll_t devx_async_cmd_event_poll(struct file *filp,
1699                                               struct poll_table_struct *wait)
1700 {
1701         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
1702         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1703         __poll_t pollflags = 0;
1704
1705         poll_wait(filp, &ev_queue->poll_wait, wait);
1706
1707         spin_lock_irq(&ev_queue->lock);
1708         if (ev_queue->is_destroyed)
1709                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1710         else if (!list_empty(&ev_queue->event_list))
1711                 pollflags = EPOLLIN | EPOLLRDNORM;
1712         spin_unlock_irq(&ev_queue->lock);
1713
1714         return pollflags;
1715 }
1716
1717 static const struct file_operations devx_async_cmd_event_fops = {
1718         .owner   = THIS_MODULE,
1719         .read    = devx_async_cmd_event_read,
1720         .poll    = devx_async_cmd_event_poll,
1721         .release = devx_async_cmd_event_close,
1722         .llseek  = no_llseek,
1723 };
1724
1725 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
1726                                                    enum rdma_remove_reason why)
1727 {
1728         struct devx_async_cmd_event_file *comp_ev_file =
1729                 container_of(uobj, struct devx_async_cmd_event_file,
1730                              uobj);
1731         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
1732
1733         spin_lock_irq(&ev_queue->lock);
1734         ev_queue->is_destroyed = 1;
1735         spin_unlock_irq(&ev_queue->lock);
1736
1737         if (why == RDMA_REMOVE_DRIVER_REMOVE)
1738                 wake_up_interruptible(&ev_queue->poll_wait);
1739
1740         mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
1741         return 0;
1742 };
1743
1744 DECLARE_UVERBS_NAMED_METHOD(
1745         MLX5_IB_METHOD_DEVX_UMEM_REG,
1746         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
1747                         MLX5_IB_OBJECT_DEVX_UMEM,
1748                         UVERBS_ACCESS_NEW,
1749                         UA_MANDATORY),
1750         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
1751                            UVERBS_ATTR_TYPE(u64),
1752                            UA_MANDATORY),
1753         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
1754                            UVERBS_ATTR_TYPE(u64),
1755                            UA_MANDATORY),
1756         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1757                              enum ib_access_flags),
1758         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
1759                             UVERBS_ATTR_TYPE(u32),
1760                             UA_MANDATORY));
1761
1762 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1763         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
1764         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
1765                         MLX5_IB_OBJECT_DEVX_UMEM,
1766                         UVERBS_ACCESS_DESTROY,
1767                         UA_MANDATORY));
1768
1769 DECLARE_UVERBS_NAMED_METHOD(
1770         MLX5_IB_METHOD_DEVX_QUERY_EQN,
1771         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
1772                            UVERBS_ATTR_TYPE(u32),
1773                            UA_MANDATORY),
1774         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1775                             UVERBS_ATTR_TYPE(u32),
1776                             UA_MANDATORY));
1777
1778 DECLARE_UVERBS_NAMED_METHOD(
1779         MLX5_IB_METHOD_DEVX_QUERY_UAR,
1780         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
1781                            UVERBS_ATTR_TYPE(u32),
1782                            UA_MANDATORY),
1783         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1784                             UVERBS_ATTR_TYPE(u32),
1785                             UA_MANDATORY));
1786
1787 DECLARE_UVERBS_NAMED_METHOD(
1788         MLX5_IB_METHOD_DEVX_OTHER,
1789         UVERBS_ATTR_PTR_IN(
1790                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
1791                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1792                 UA_MANDATORY,
1793                 UA_ALLOC_AND_COPY),
1794         UVERBS_ATTR_PTR_OUT(
1795                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
1796                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1797                 UA_MANDATORY));
1798
1799 DECLARE_UVERBS_NAMED_METHOD(
1800         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
1801         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
1802                         MLX5_IB_OBJECT_DEVX_OBJ,
1803                         UVERBS_ACCESS_NEW,
1804                         UA_MANDATORY),
1805         UVERBS_ATTR_PTR_IN(
1806                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
1807                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1808                 UA_MANDATORY,
1809                 UA_ALLOC_AND_COPY),
1810         UVERBS_ATTR_PTR_OUT(
1811                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1812                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1813                 UA_MANDATORY));
1814
1815 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1816         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
1817         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
1818                         MLX5_IB_OBJECT_DEVX_OBJ,
1819                         UVERBS_ACCESS_DESTROY,
1820                         UA_MANDATORY));
1821
1822 DECLARE_UVERBS_NAMED_METHOD(
1823         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
1824         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
1825                         UVERBS_IDR_ANY_OBJECT,
1826                         UVERBS_ACCESS_WRITE,
1827                         UA_MANDATORY),
1828         UVERBS_ATTR_PTR_IN(
1829                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
1830                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1831                 UA_MANDATORY,
1832                 UA_ALLOC_AND_COPY),
1833         UVERBS_ATTR_PTR_OUT(
1834                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1835                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1836                 UA_MANDATORY));
1837
1838 DECLARE_UVERBS_NAMED_METHOD(
1839         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
1840         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
1841                         UVERBS_IDR_ANY_OBJECT,
1842                         UVERBS_ACCESS_READ,
1843                         UA_MANDATORY),
1844         UVERBS_ATTR_PTR_IN(
1845                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
1846                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1847                 UA_MANDATORY,
1848                 UA_ALLOC_AND_COPY),
1849         UVERBS_ATTR_PTR_OUT(
1850                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1851                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1852                 UA_MANDATORY));
1853
1854 DECLARE_UVERBS_NAMED_METHOD(
1855         MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
1856         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
1857                         UVERBS_IDR_ANY_OBJECT,
1858                         UVERBS_ACCESS_READ,
1859                         UA_MANDATORY),
1860         UVERBS_ATTR_PTR_IN(
1861                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
1862                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1863                 UA_MANDATORY,
1864                 UA_ALLOC_AND_COPY),
1865         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
1866                 u16, UA_MANDATORY),
1867         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
1868                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1869                 UVERBS_ACCESS_READ,
1870                 UA_MANDATORY),
1871         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
1872                 UVERBS_ATTR_TYPE(u64),
1873                 UA_MANDATORY));
1874
1875 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
1876                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
1877                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
1878                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN));
1879
1880 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
1881                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
1882                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
1883                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
1884                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
1885                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
1886                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
1887
1888 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
1889                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
1890                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
1891                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
1892
1893
1894 DECLARE_UVERBS_NAMED_METHOD(
1895         MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
1896         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
1897                         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1898                         UVERBS_ACCESS_NEW,
1899                         UA_MANDATORY));
1900
1901 DECLARE_UVERBS_NAMED_OBJECT(
1902         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1903         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
1904                              devx_hot_unplug_async_cmd_event_file,
1905                              &devx_async_cmd_event_fops, "[devx_async_cmd]",
1906                              O_RDONLY),
1907         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
1908
1909 static bool devx_is_supported(struct ib_device *device)
1910 {
1911         struct mlx5_ib_dev *dev = to_mdev(device);
1912
1913         return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
1914 }
1915
1916 const struct uapi_definition mlx5_ib_devx_defs[] = {
1917         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1918                 MLX5_IB_OBJECT_DEVX,
1919                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1920         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1921                 MLX5_IB_OBJECT_DEVX_OBJ,
1922                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1923         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1924                 MLX5_IB_OBJECT_DEVX_UMEM,
1925                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1926         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1927                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
1928                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1929         {},
1930 };