drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/mlx5_user_ioctl_verbs.h>
  12 #include <rdma/ib_umem.h>
  13 #include <rdma/uverbs_std_types.h>
  14 #include <linux/mlx5/driver.h>
  15 #include <linux/mlx5/fs.h>
  16 #include "mlx5_ib.h"
  17 #include "devx.h"
  18 #include "qp.h"
  19 #include <linux/xarray.h>
  20
  21 #define UVERBS_MODULE_NAME mlx5_ib
  22 #include <rdma/uverbs_named_ioctl.h>
  23
  24 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
  25
  26 enum devx_obj_flags {
  27         DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  28         DEVX_OBJ_FLAGS_DCT = 1 << 1,
  29         DEVX_OBJ_FLAGS_CQ = 1 << 2,
  30 };
  31
  32 struct devx_async_data {
  33         struct mlx5_ib_dev *mdev;
  34         struct list_head list;
  35         struct devx_async_cmd_event_file *ev_file;
  36         struct mlx5_async_work cb_work;
  37         u16 cmd_out_len;
  38         /* must be last field in this structure */
  39         struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  40 };
  41
  42 struct devx_async_event_data {
  43         struct list_head list; /* headed in ev_file->event_list */
  44         struct mlx5_ib_uapi_devx_async_event_hdr hdr;
  45 };
  46
  47 /* first level XA value data structure */
  48 struct devx_event {
  49         struct xarray object_ids; /* second XA level, Key = object id */
  50         struct list_head unaffiliated_list;
  51 };
  52
  53 /* second level XA value data structure */
  54 struct devx_obj_event {
  55         struct rcu_head rcu;
  56         struct list_head obj_sub_list;
  57 };
  58
  59 struct devx_event_subscription {
  60         struct list_head file_list; /* headed in ev_file->
  61                                      * subscribed_events_list
  62                                      */
  63         struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
  64                                    * devx_obj_event->obj_sub_list
  65                                    */
  66         struct list_head obj_list; /* headed in devx_object */
  67         struct list_head event_list; /* headed in ev_file->event_list or in
  68                                       * temp list via subscription
  69                                       */
  70
  71         u8 is_cleaned:1;
  72         u32 xa_key_level1;
  73         u32 xa_key_level2;
  74         struct rcu_head rcu;
  75         u64 cookie;
  76         struct devx_async_event_file *ev_file;
  77         struct eventfd_ctx *eventfd;
  78 };
  79
  80 struct devx_async_event_file {
  81         struct ib_uobject uobj;
  82         /* Head of events that are subscribed to this FD */
  83         struct list_head subscribed_events_list;
  84         spinlock_t lock;
  85         wait_queue_head_t poll_wait;
  86         struct list_head event_list;
  87         struct mlx5_ib_dev *dev;
  88         u8 omit_data:1;
  89         u8 is_overflow_err:1;
  90         u8 is_destroyed:1;
  91 };
  92
  93 struct devx_umem {
  94         struct mlx5_core_dev            *mdev;
  95         struct ib_umem                  *umem;
  96         u32                             dinlen;
  97         u32                             dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];
  98 };
  99
 100 struct devx_umem_reg_cmd {
 101         void                            *in;
 102         u32                             inlen;
 103         u32                             out[MLX5_ST_SZ_DW(create_umem_out)];
 104 };
 105
 106 static struct mlx5_ib_ucontext *
 107 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
 108 {
 109         return to_mucontext(ib_uverbs_get_ucontext(attrs));
 110 }
 111
 112 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
 113 {
 114         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};
 115         u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
 116         void *uctx;
 117         int err;
 118         u16 uid;
 119         u32 cap = 0;
 120
 121         /* 0 means not supported */
 122         if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
 123                 return -EINVAL;
 124
 125         uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
 126         if (is_user && capable(CAP_NET_RAW) &&
 127             (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
 128                 cap |= MLX5_UCTX_CAP_RAW_TX;
 129         if (is_user && capable(CAP_SYS_RAWIO) &&
 130             (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
 131              MLX5_UCTX_CAP_INTERNAL_DEV_RES))
 132                 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
 133
 134         MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
 135         MLX5_SET(uctx, uctx, cap, cap);
 136
 137         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 138         if (err)
 139                 return err;
 140
 141         uid = MLX5_GET(create_uctx_out, out, uid);
 142         return uid;
 143 }
 144
 145 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 146 {
 147         u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
 148         u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
 149
 150         MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 151         MLX5_SET(destroy_uctx_in, in, uid, uid);
 152
 153         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 154 }
 155
 156 static bool is_legacy_unaffiliated_event_num(u16 event_num)
 157 {
 158         switch (event_num) {
 159         case MLX5_EVENT_TYPE_PORT_CHANGE:
 160                 return true;
 161         default:
 162                 return false;
 163         }
 164 }
 165
 166 static bool is_legacy_obj_event_num(u16 event_num)
 167 {
 168         switch (event_num) {
 169         case MLX5_EVENT_TYPE_PATH_MIG:
 170         case MLX5_EVENT_TYPE_COMM_EST:
 171         case MLX5_EVENT_TYPE_SQ_DRAINED:
 172         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 173         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 174         case MLX5_EVENT_TYPE_CQ_ERROR:
 175         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 176         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 177         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 178         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 179         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 180         case MLX5_EVENT_TYPE_DCT_DRAINED:
 181         case MLX5_EVENT_TYPE_COMP:
 182         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 183         case MLX5_EVENT_TYPE_XRQ_ERROR:
 184                 return true;
 185         default:
 186                 return false;
 187         }
 188 }
 189
 190 static u16 get_legacy_obj_type(u16 opcode)
 191 {
 192         switch (opcode) {
 193         case MLX5_CMD_OP_CREATE_RQ:
 194                 return MLX5_EVENT_QUEUE_TYPE_RQ;
 195         case MLX5_CMD_OP_CREATE_QP:
 196                 return MLX5_EVENT_QUEUE_TYPE_QP;
 197         case MLX5_CMD_OP_CREATE_SQ:
 198                 return MLX5_EVENT_QUEUE_TYPE_SQ;
 199         case MLX5_CMD_OP_CREATE_DCT:
 200                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 201         default:
 202                 return 0;
 203         }
 204 }
 205
 206 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
 207 {
 208         u16 opcode;
 209
 210         opcode = (obj->obj_id >> 32) & 0xffff;
 211
 212         if (is_legacy_obj_event_num(event_num))
 213                 return get_legacy_obj_type(opcode);
 214
 215         switch (opcode) {
 216         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 217                 return (obj->obj_id >> 48);
 218         case MLX5_CMD_OP_CREATE_RQ:
 219                 return MLX5_OBJ_TYPE_RQ;
 220         case MLX5_CMD_OP_CREATE_QP:
 221                 return MLX5_OBJ_TYPE_QP;
 222         case MLX5_CMD_OP_CREATE_SQ:
 223                 return MLX5_OBJ_TYPE_SQ;
 224         case MLX5_CMD_OP_CREATE_DCT:
 225                 return MLX5_OBJ_TYPE_DCT;
 226         case MLX5_CMD_OP_CREATE_TIR:
 227                 return MLX5_OBJ_TYPE_TIR;
 228         case MLX5_CMD_OP_CREATE_TIS:
 229                 return MLX5_OBJ_TYPE_TIS;
 230         case MLX5_CMD_OP_CREATE_PSV:
 231                 return MLX5_OBJ_TYPE_PSV;
 232         case MLX5_OBJ_TYPE_MKEY:
 233                 return MLX5_OBJ_TYPE_MKEY;
 234         case MLX5_CMD_OP_CREATE_RMP:
 235                 return MLX5_OBJ_TYPE_RMP;
 236         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 237                 return MLX5_OBJ_TYPE_XRC_SRQ;
 238         case MLX5_CMD_OP_CREATE_XRQ:
 239                 return MLX5_OBJ_TYPE_XRQ;
 240         case MLX5_CMD_OP_CREATE_RQT:
 241                 return MLX5_OBJ_TYPE_RQT;
 242         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 243                 return MLX5_OBJ_TYPE_FLOW_COUNTER;
 244         case MLX5_CMD_OP_CREATE_CQ:
 245                 return MLX5_OBJ_TYPE_CQ;
 246         default:
 247                 return 0;
 248         }
 249 }
 250
 251 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
 252 {
 253         switch (event_type) {
 254         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 255         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 256         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 257         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 258         case MLX5_EVENT_TYPE_PATH_MIG:
 259         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 260         case MLX5_EVENT_TYPE_COMM_EST:
 261         case MLX5_EVENT_TYPE_SQ_DRAINED:
 262         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 263         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 264                 return eqe->data.qp_srq.type;
 265         case MLX5_EVENT_TYPE_CQ_ERROR:
 266         case MLX5_EVENT_TYPE_XRQ_ERROR:
 267                 return 0;
 268         case MLX5_EVENT_TYPE_DCT_DRAINED:
 269         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 270                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 271         default:
 272                 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
 273         }
 274 }
 275
 276 static u32 get_dec_obj_id(u64 obj_id)
 277 {
 278         return (obj_id & 0xffffffff);
 279 }
 280
 281 /*
 282  * As the obj_id in the firmware is not globally unique the object type
 283  * must be considered upon checking for a valid object id.
 284  * For that the opcode of the creator command is encoded as part of the obj_id.
 285  */
 286 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
 287 {
 288         return ((u64)opcode << 32) | obj_id;
 289 }
 290
 291 static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)
 292 {
 293         switch (opcode) {
 294         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 295                 return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 296         case MLX5_CMD_OP_CREATE_UMEM:
 297                 return MLX5_GET(create_umem_out, out, umem_id);
 298         case MLX5_CMD_OP_CREATE_MKEY:
 299                 return MLX5_GET(create_mkey_out, out, mkey_index);
 300         case MLX5_CMD_OP_CREATE_CQ:
 301                 return MLX5_GET(create_cq_out, out, cqn);
 302         case MLX5_CMD_OP_ALLOC_PD:
 303                 return MLX5_GET(alloc_pd_out, out, pd);
 304         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 305                 return MLX5_GET(alloc_transport_domain_out, out,
 306                                 transport_domain);
 307         case MLX5_CMD_OP_CREATE_RMP:
 308                 return MLX5_GET(create_rmp_out, out, rmpn);
 309         case MLX5_CMD_OP_CREATE_SQ:
 310                 return MLX5_GET(create_sq_out, out, sqn);
 311         case MLX5_CMD_OP_CREATE_RQ:
 312                 return MLX5_GET(create_rq_out, out, rqn);
 313         case MLX5_CMD_OP_CREATE_RQT:
 314                 return MLX5_GET(create_rqt_out, out, rqtn);
 315         case MLX5_CMD_OP_CREATE_TIR:
 316                 return MLX5_GET(create_tir_out, out, tirn);
 317         case MLX5_CMD_OP_CREATE_TIS:
 318                 return MLX5_GET(create_tis_out, out, tisn);
 319         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 320                 return MLX5_GET(alloc_q_counter_out, out, counter_set_id);
 321         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 322                 return MLX5_GET(create_flow_table_out, out, table_id);
 323         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 324                 return MLX5_GET(create_flow_group_out, out, group_id);
 325         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 326                 return MLX5_GET(set_fte_in, in, flow_index);
 327         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 328                 return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
 329         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 330                 return MLX5_GET(alloc_packet_reformat_context_out, out,
 331                                 packet_reformat_id);
 332         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 333                 return MLX5_GET(alloc_modify_header_context_out, out,
 334                                 modify_header_id);
 335         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 336                 return MLX5_GET(create_scheduling_element_out, out,
 337                                 scheduling_element_id);
 338         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 339                 return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
 340         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 341                 return MLX5_GET(set_l2_table_entry_in, in, table_index);
 342         case MLX5_CMD_OP_CREATE_QP:
 343                 return MLX5_GET(create_qp_out, out, qpn);
 344         case MLX5_CMD_OP_CREATE_SRQ:
 345                 return MLX5_GET(create_srq_out, out, srqn);
 346         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 347                 return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
 348         case MLX5_CMD_OP_CREATE_DCT:
 349                 return MLX5_GET(create_dct_out, out, dctn);
 350         case MLX5_CMD_OP_CREATE_XRQ:
 351                 return MLX5_GET(create_xrq_out, out, xrqn);
 352         case MLX5_CMD_OP_ATTACH_TO_MCG:
 353                 return MLX5_GET(attach_to_mcg_in, in, qpn);
 354         case MLX5_CMD_OP_ALLOC_XRCD:
 355                 return MLX5_GET(alloc_xrcd_out, out, xrcd);
 356         case MLX5_CMD_OP_CREATE_PSV:
 357                 return MLX5_GET(create_psv_out, out, psv0_index);
 358         default:
 359                 /* The entry must match to one of the devx_is_obj_create_cmd */
 360                 WARN_ON(true);
 361                 return 0;
 362         }
 363 }
 364
 365 static u64 devx_get_obj_id(const void *in)
 366 {
 367         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 368         u64 obj_id;
 369
 370         switch (opcode) {
 371         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 372         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 373                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
 374                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 375                                                  obj_type) << 16,
 376                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 377                                                  obj_id));
 378                 break;
 379         case MLX5_CMD_OP_QUERY_MKEY:
 380                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 381                                         MLX5_GET(query_mkey_in, in,
 382                                                  mkey_index));
 383                 break;
 384         case MLX5_CMD_OP_QUERY_CQ:
 385                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 386                                         MLX5_GET(query_cq_in, in, cqn));
 387                 break;
 388         case MLX5_CMD_OP_MODIFY_CQ:
 389                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 390                                         MLX5_GET(modify_cq_in, in, cqn));
 391                 break;
 392         case MLX5_CMD_OP_QUERY_SQ:
 393                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 394                                         MLX5_GET(query_sq_in, in, sqn));
 395                 break;
 396         case MLX5_CMD_OP_MODIFY_SQ:
 397                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 398                                         MLX5_GET(modify_sq_in, in, sqn));
 399                 break;
 400         case MLX5_CMD_OP_QUERY_RQ:
 401                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 402                                         MLX5_GET(query_rq_in, in, rqn));
 403                 break;
 404         case MLX5_CMD_OP_MODIFY_RQ:
 405                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 406                                         MLX5_GET(modify_rq_in, in, rqn));
 407                 break;
 408         case MLX5_CMD_OP_QUERY_RMP:
 409                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 410                                         MLX5_GET(query_rmp_in, in, rmpn));
 411                 break;
 412         case MLX5_CMD_OP_MODIFY_RMP:
 413                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 414                                         MLX5_GET(modify_rmp_in, in, rmpn));
 415                 break;
 416         case MLX5_CMD_OP_QUERY_RQT:
 417                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 418                                         MLX5_GET(query_rqt_in, in, rqtn));
 419                 break;
 420         case MLX5_CMD_OP_MODIFY_RQT:
 421                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 422                                         MLX5_GET(modify_rqt_in, in, rqtn));
 423                 break;
 424         case MLX5_CMD_OP_QUERY_TIR:
 425                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 426                                         MLX5_GET(query_tir_in, in, tirn));
 427                 break;
 428         case MLX5_CMD_OP_MODIFY_TIR:
 429                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 430                                         MLX5_GET(modify_tir_in, in, tirn));
 431                 break;
 432         case MLX5_CMD_OP_QUERY_TIS:
 433                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 434                                         MLX5_GET(query_tis_in, in, tisn));
 435                 break;
 436         case MLX5_CMD_OP_MODIFY_TIS:
 437                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 438                                         MLX5_GET(modify_tis_in, in, tisn));
 439                 break;
 440         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 441                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 442                                         MLX5_GET(query_flow_table_in, in,
 443                                                  table_id));
 444                 break;
 445         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 446                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 447                                         MLX5_GET(modify_flow_table_in, in,
 448                                                  table_id));
 449                 break;
 450         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 451                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 452                                         MLX5_GET(query_flow_group_in, in,
 453                                                  group_id));
 454                 break;
 455         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 456                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 457                                         MLX5_GET(query_fte_in, in,
 458                                                  flow_index));
 459                 break;
 460         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 461                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 462                                         MLX5_GET(set_fte_in, in, flow_index));
 463                 break;
 464         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 465                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 466                                         MLX5_GET(query_q_counter_in, in,
 467                                                  counter_set_id));
 468                 break;
 469         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 470                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 471                                         MLX5_GET(query_flow_counter_in, in,
 472                                                  flow_counter_id));
 473                 break;
 474         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 475                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 476                                         MLX5_GET(query_modify_header_context_in,
 477                                                  in, modify_header_id));
 478                 break;
 479         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 480                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 481                                         MLX5_GET(query_scheduling_element_in,
 482                                                  in, scheduling_element_id));
 483                 break;
 484         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 485                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 486                                         MLX5_GET(modify_scheduling_element_in,
 487                                                  in, scheduling_element_id));
 488                 break;
 489         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 490                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 491                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 492                                                  vxlan_udp_port));
 493                 break;
 494         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 495                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 496                                         MLX5_GET(query_l2_table_entry_in, in,
 497                                                  table_index));
 498                 break;
 499         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 500                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 501                                         MLX5_GET(set_l2_table_entry_in, in,
 502                                                  table_index));
 503                 break;
 504         case MLX5_CMD_OP_QUERY_QP:
 505                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 506                                         MLX5_GET(query_qp_in, in, qpn));
 507                 break;
 508         case MLX5_CMD_OP_RST2INIT_QP:
 509                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 510                                         MLX5_GET(rst2init_qp_in, in, qpn));
 511                 break;
 512         case MLX5_CMD_OP_INIT2INIT_QP:
 513                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 514                                         MLX5_GET(init2init_qp_in, in, qpn));
 515                 break;
 516         case MLX5_CMD_OP_INIT2RTR_QP:
 517                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 518                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 519                 break;
 520         case MLX5_CMD_OP_RTR2RTS_QP:
 521                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 522                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 523                 break;
 524         case MLX5_CMD_OP_RTS2RTS_QP:
 525                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 526                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 527                 break;
 528         case MLX5_CMD_OP_SQERR2RTS_QP:
 529                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 530                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 531                 break;
 532         case MLX5_CMD_OP_2ERR_QP:
 533                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 534                                         MLX5_GET(qp_2err_in, in, qpn));
 535                 break;
 536         case MLX5_CMD_OP_2RST_QP:
 537                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 538                                         MLX5_GET(qp_2rst_in, in, qpn));
 539                 break;
 540         case MLX5_CMD_OP_QUERY_DCT:
 541                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 542                                         MLX5_GET(query_dct_in, in, dctn));
 543                 break;
 544         case MLX5_CMD_OP_QUERY_XRQ:
 545         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 546         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 547                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 548                                         MLX5_GET(query_xrq_in, in, xrqn));
 549                 break;
 550         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 551                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 552                                         MLX5_GET(query_xrc_srq_in, in,
 553                                                  xrc_srqn));
 554                 break;
 555         case MLX5_CMD_OP_ARM_XRC_SRQ:
 556                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 557                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 558                 break;
 559         case MLX5_CMD_OP_QUERY_SRQ:
 560                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 561                                         MLX5_GET(query_srq_in, in, srqn));
 562                 break;
 563         case MLX5_CMD_OP_ARM_RQ:
 564                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 565                                         MLX5_GET(arm_rq_in, in, srq_number));
 566                 break;
 567         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 568                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 569                                         MLX5_GET(drain_dct_in, in, dctn));
 570                 break;
 571         case MLX5_CMD_OP_ARM_XRQ:
 572         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 573         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 574         case MLX5_CMD_OP_MODIFY_XRQ:
 575                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 576                                         MLX5_GET(arm_xrq_in, in, xrqn));
 577                 break;
 578         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 579                 obj_id = get_enc_obj_id
 580                                 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 581                                  MLX5_GET(query_packet_reformat_context_in,
 582                                           in, packet_reformat_id));
 583                 break;
 584         default:
 585                 obj_id = 0;
 586         }
 587
 588         return obj_id;
 589 }
 590
 591 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 592                                  struct ib_uobject *uobj, const void *in)
 593 {
 594         struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 595         u64 obj_id = devx_get_obj_id(in);
 596
 597         if (!obj_id)
 598                 return false;
 599
 600         switch (uobj_get_object_id(uobj)) {
 601         case UVERBS_OBJECT_CQ:
 602                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 603                                       to_mcq(uobj->object)->mcq.cqn) ==
 604                                       obj_id;
 605
 606         case UVERBS_OBJECT_SRQ:
 607         {
 608                 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 609                 u16 opcode;
 610
 611                 switch (srq->common.res) {
 612                 case MLX5_RES_XSRQ:
 613                         opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 614                         break;
 615                 case MLX5_RES_XRQ:
 616                         opcode = MLX5_CMD_OP_CREATE_XRQ;
 617                         break;
 618                 default:
 619                         if (!dev->mdev->issi)
 620                                 opcode = MLX5_CMD_OP_CREATE_SRQ;
 621                         else
 622                                 opcode = MLX5_CMD_OP_CREATE_RMP;
 623                 }
 624
 625                 return get_enc_obj_id(opcode,
 626                                       to_msrq(uobj->object)->msrq.srqn) ==
 627                                       obj_id;
 628         }
 629
 630         case UVERBS_OBJECT_QP:
 631         {
 632                 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 633
 634                 if (qp->type == IB_QPT_RAW_PACKET ||
 635                     (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
 636                         struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 637                                                          &qp->raw_packet_qp;
 638                         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 639                         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 640
 641                         return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 642                                                rq->base.mqp.qpn) == obj_id ||
 643                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 644                                                sq->base.mqp.qpn) == obj_id ||
 645                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 646                                                rq->tirn) == obj_id ||
 647                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 648                                                sq->tisn) == obj_id);
 649                 }
 650
 651                 if (qp->type == MLX5_IB_QPT_DCT)
 652                         return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 653                                               qp->dct.mdct.mqp.qpn) == obj_id;
 654                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 655                                       qp->ibqp.qp_num) == obj_id;
 656         }
 657
 658         case UVERBS_OBJECT_WQ:
 659                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 660                                       to_mrwq(uobj->object)->core_qp.qpn) ==
 661                                       obj_id;
 662
 663         case UVERBS_OBJECT_RWQ_IND_TBL:
 664                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 665                                       to_mrwq_ind_table(uobj->object)->rqtn) ==
 666                                       obj_id;
 667
 668         case MLX5_IB_OBJECT_DEVX_OBJ:
 669                 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
 670
 671         default:
 672                 return false;
 673         }
 674 }
 675
 676 static void devx_set_umem_valid(const void *in)
 677 {
 678         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 679
 680         switch (opcode) {
 681         case MLX5_CMD_OP_CREATE_MKEY:
 682                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 683                 break;
 684         case MLX5_CMD_OP_CREATE_CQ:
 685         {
 686                 void *cqc;
 687
 688                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 689                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 690                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 691                 break;
 692         }
 693         case MLX5_CMD_OP_CREATE_QP:
 694         {
 695                 void *qpc;
 696
 697                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 698                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 699                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 700                 break;
 701         }
 702
 703         case MLX5_CMD_OP_CREATE_RQ:
 704         {
 705                 void *rqc, *wq;
 706
 707                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 708                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 709                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 710                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 711                 break;
 712         }
 713
 714         case MLX5_CMD_OP_CREATE_SQ:
 715         {
 716                 void *sqc, *wq;
 717
 718                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 719                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 720                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 721                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 722                 break;
 723         }
 724
 725         case MLX5_CMD_OP_MODIFY_CQ:
 726                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 727                 break;
 728
 729         case MLX5_CMD_OP_CREATE_RMP:
 730         {
 731                 void *rmpc, *wq;
 732
 733                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 734                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 735                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 736                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 737                 break;
 738         }
 739
 740         case MLX5_CMD_OP_CREATE_XRQ:
 741         {
 742                 void *xrqc, *wq;
 743
 744                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 745                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 746                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 747                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 748                 break;
 749         }
 750
 751         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 752         {
 753                 void *xrc_srqc;
 754
 755                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 756                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 757                                         xrc_srq_context_entry);
 758                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 759                 break;
 760         }
 761
 762         default:
 763                 return;
 764         }
 765 }
 766
 767 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 768 {
 769         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 770
 771         switch (*opcode) {
 772         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 773         case MLX5_CMD_OP_CREATE_MKEY:
 774         case MLX5_CMD_OP_CREATE_CQ:
 775         case MLX5_CMD_OP_ALLOC_PD:
 776         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 777         case MLX5_CMD_OP_CREATE_RMP:
 778         case MLX5_CMD_OP_CREATE_SQ:
 779         case MLX5_CMD_OP_CREATE_RQ:
 780         case MLX5_CMD_OP_CREATE_RQT:
 781         case MLX5_CMD_OP_CREATE_TIR:
 782         case MLX5_CMD_OP_CREATE_TIS:
 783         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 784         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 785         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 786         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 787         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 788         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 789         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 790         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 791         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 792         case MLX5_CMD_OP_CREATE_QP:
 793         case MLX5_CMD_OP_CREATE_SRQ:
 794         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 795         case MLX5_CMD_OP_CREATE_DCT:
 796         case MLX5_CMD_OP_CREATE_XRQ:
 797         case MLX5_CMD_OP_ATTACH_TO_MCG:
 798         case MLX5_CMD_OP_ALLOC_XRCD:
 799                 return true;
 800         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 801         {
 802                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 803                 if (op_mod == 0)
 804                         return true;
 805                 return false;
 806         }
 807         case MLX5_CMD_OP_CREATE_PSV:
 808         {
 809                 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
 810
 811                 if (num_psv == 1)
 812                         return true;
 813                 return false;
 814         }
 815         default:
 816                 return false;
 817         }
 818 }
 819
 820 static bool devx_is_obj_modify_cmd(const void *in)
 821 {
 822         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 823
 824         switch (opcode) {
 825         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 826         case MLX5_CMD_OP_MODIFY_CQ:
 827         case MLX5_CMD_OP_MODIFY_RMP:
 828         case MLX5_CMD_OP_MODIFY_SQ:
 829         case MLX5_CMD_OP_MODIFY_RQ:
 830         case MLX5_CMD_OP_MODIFY_RQT:
 831         case MLX5_CMD_OP_MODIFY_TIR:
 832         case MLX5_CMD_OP_MODIFY_TIS:
 833         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 834         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 835         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 836         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 837         case MLX5_CMD_OP_RST2INIT_QP:
 838         case MLX5_CMD_OP_INIT2RTR_QP:
 839         case MLX5_CMD_OP_INIT2INIT_QP:
 840         case MLX5_CMD_OP_RTR2RTS_QP:
 841         case MLX5_CMD_OP_RTS2RTS_QP:
 842         case MLX5_CMD_OP_SQERR2RTS_QP:
 843         case MLX5_CMD_OP_2ERR_QP:
 844         case MLX5_CMD_OP_2RST_QP:
 845         case MLX5_CMD_OP_ARM_XRC_SRQ:
 846         case MLX5_CMD_OP_ARM_RQ:
 847         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 848         case MLX5_CMD_OP_ARM_XRQ:
 849         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 850         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 851         case MLX5_CMD_OP_MODIFY_XRQ:
 852                 return true;
 853         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 854         {
 855                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 856
 857                 if (op_mod == 1)
 858                         return true;
 859                 return false;
 860         }
 861         default:
 862                 return false;
 863         }
 864 }
 865
 866 static bool devx_is_obj_query_cmd(const void *in)
 867 {
 868         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 869
 870         switch (opcode) {
 871         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 872         case MLX5_CMD_OP_QUERY_MKEY:
 873         case MLX5_CMD_OP_QUERY_CQ:
 874         case MLX5_CMD_OP_QUERY_RMP:
 875         case MLX5_CMD_OP_QUERY_SQ:
 876         case MLX5_CMD_OP_QUERY_RQ:
 877         case MLX5_CMD_OP_QUERY_RQT:
 878         case MLX5_CMD_OP_QUERY_TIR:
 879         case MLX5_CMD_OP_QUERY_TIS:
 880         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 881         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 882         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 883         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 884         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 885         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 886         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 887         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 888         case MLX5_CMD_OP_QUERY_QP:
 889         case MLX5_CMD_OP_QUERY_SRQ:
 890         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 891         case MLX5_CMD_OP_QUERY_DCT:
 892         case MLX5_CMD_OP_QUERY_XRQ:
 893         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 894         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 895         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 896                 return true;
 897         default:
 898                 return false;
 899         }
 900 }
 901
 902 static bool devx_is_whitelist_cmd(void *in)
 903 {
 904         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 905
 906         switch (opcode) {
 907         case MLX5_CMD_OP_QUERY_HCA_CAP:
 908         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 909         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 910                 return true;
 911         default:
 912                 return false;
 913         }
 914 }
 915
 916 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 917 {
 918         if (devx_is_whitelist_cmd(cmd_in)) {
 919                 struct mlx5_ib_dev *dev;
 920
 921                 if (c->devx_uid)
 922                         return c->devx_uid;
 923
 924                 dev = to_mdev(c->ibucontext.device);
 925                 if (dev->devx_whitelist_uid)
 926                         return dev->devx_whitelist_uid;
 927
 928                 return -EOPNOTSUPP;
 929         }
 930
 931         if (!c->devx_uid)
 932                 return -EINVAL;
 933
 934         return c->devx_uid;
 935 }
 936
 937 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
 938 {
 939         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 940
 941         /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
 942         if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
 943              MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
 944             (opcode >= MLX5_CMD_OP_GENERAL_START &&
 945              opcode < MLX5_CMD_OP_GENERAL_END))
 946                 return true;
 947
 948         switch (opcode) {
 949         case MLX5_CMD_OP_QUERY_HCA_CAP:
 950         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 951         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 952         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 953         case MLX5_CMD_OP_QUERY_ADAPTER:
 954         case MLX5_CMD_OP_QUERY_ISSI:
 955         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 956         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 957         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 958         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 959         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 960         case MLX5_CMD_OP_NOP:
 961         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 962         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 963         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 964         case MLX5_CMD_OP_QUERY_LAG:
 965                 return true;
 966         default:
 967                 return false;
 968         }
 969 }
 970
 971 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 972         struct uverbs_attr_bundle *attrs)
 973 {
 974         struct mlx5_ib_ucontext *c;
 975         struct mlx5_ib_dev *dev;
 976         int user_vector;
 977         int dev_eqn;
 978         int err;
 979
 980         if (uverbs_copy_from(&user_vector, attrs,
 981                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 982                 return -EFAULT;
 983
 984         c = devx_ufile2uctx(attrs);
 985         if (IS_ERR(c))
 986                 return PTR_ERR(c);
 987         dev = to_mdev(c->ibucontext.device);
 988
 989         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn);
 990         if (err < 0)
 991                 return err;
 992
 993         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 994                            &dev_eqn, sizeof(dev_eqn)))
 995                 return -EFAULT;
 996
 997         return 0;
 998 }
 999
1000 /*
1001  *Security note:
1002  * The hardware protection mechanism works like this: Each device object that
1003  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
1004  * the device specification manual) upon its creation. Then upon doorbell,
1005  * hardware fetches the object context for which the doorbell was rang, and
1006  * validates that the UAR through which the DB was rang matches the UAR ID
1007  * of the object.
1008  * If no match the doorbell is silently ignored by the hardware. Of course,
1009  * the user cannot ring a doorbell on a UAR that was not mapped to it.
1010  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1011  * mailboxes (except tagging them with UID), we expose to the user its UAR
1012  * ID, so it can embed it in these objects in the expected specification
1013  * format. So the only thing the user can do is hurt itself by creating a
1014  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1015  * may ring a doorbell on its objects.
1016  * The consequence of that will be that another user can schedule a QP/SQ
1017  * of the buggy user for execution (just insert it to the hardware schedule
1018  * queue or arm its CQ for event generation), no further harm is expected.
1019  */
1020 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1021         struct uverbs_attr_bundle *attrs)
1022 {
1023         struct mlx5_ib_ucontext *c;
1024         struct mlx5_ib_dev *dev;
1025         u32 user_idx;
1026         s32 dev_idx;
1027
1028         c = devx_ufile2uctx(attrs);
1029         if (IS_ERR(c))
1030                 return PTR_ERR(c);
1031         dev = to_mdev(c->ibucontext.device);
1032
1033         if (uverbs_copy_from(&user_idx, attrs,
1034                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1035                 return -EFAULT;
1036
1037         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1038         if (dev_idx < 0)
1039                 return dev_idx;
1040
1041         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1042                            &dev_idx, sizeof(dev_idx)))
1043                 return -EFAULT;
1044
1045         return 0;
1046 }
1047
1048 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1049         struct uverbs_attr_bundle *attrs)
1050 {
1051         struct mlx5_ib_ucontext *c;
1052         struct mlx5_ib_dev *dev;
1053         void *cmd_in = uverbs_attr_get_alloced_ptr(
1054                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1055         int cmd_out_len = uverbs_attr_get_len(attrs,
1056                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1057         void *cmd_out;
1058         int err;
1059         int uid;
1060
1061         c = devx_ufile2uctx(attrs);
1062         if (IS_ERR(c))
1063                 return PTR_ERR(c);
1064         dev = to_mdev(c->ibucontext.device);
1065
1066         uid = devx_get_uid(c, cmd_in);
1067         if (uid < 0)
1068                 return uid;
1069
1070         /* Only white list of some general HCA commands are allowed for this method. */
1071         if (!devx_is_general_cmd(cmd_in, dev))
1072                 return -EINVAL;
1073
1074         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1075         if (IS_ERR(cmd_out))
1076                 return PTR_ERR(cmd_out);
1077
1078         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1079         err = mlx5_cmd_exec(dev->mdev, cmd_in,
1080                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1081                             cmd_out, cmd_out_len);
1082         if (err)
1083                 return err;
1084
1085         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1086                               cmd_out_len);
1087 }
1088
1089 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1090                                        u32 *dinlen,
1091                                        u32 *obj_id)
1092 {
1093         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
1094         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1095
1096         *obj_id = devx_get_created_obj_id(in, out, opcode);
1097         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1098         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1099
1100         switch (opcode) {
1101         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1102                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1103                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1104                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,
1105                          MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));
1106                 break;
1107
1108         case MLX5_CMD_OP_CREATE_UMEM:
1109                 MLX5_SET(destroy_umem_in, din, opcode,
1110                          MLX5_CMD_OP_DESTROY_UMEM);
1111                 MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);
1112                 break;
1113         case MLX5_CMD_OP_CREATE_MKEY:
1114                 MLX5_SET(destroy_mkey_in, din, opcode,
1115                          MLX5_CMD_OP_DESTROY_MKEY);
1116                 MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);
1117                 break;
1118         case MLX5_CMD_OP_CREATE_CQ:
1119                 MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1120                 MLX5_SET(destroy_cq_in, din, cqn, *obj_id);
1121                 break;
1122         case MLX5_CMD_OP_ALLOC_PD:
1123                 MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1124                 MLX5_SET(dealloc_pd_in, din, pd, *obj_id);
1125                 break;
1126         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1127                 MLX5_SET(dealloc_transport_domain_in, din, opcode,
1128                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1129                 MLX5_SET(dealloc_transport_domain_in, din, transport_domain,
1130                          *obj_id);
1131                 break;
1132         case MLX5_CMD_OP_CREATE_RMP:
1133                 MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1134                 MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);
1135                 break;
1136         case MLX5_CMD_OP_CREATE_SQ:
1137                 MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1138                 MLX5_SET(destroy_sq_in, din, sqn, *obj_id);
1139                 break;
1140         case MLX5_CMD_OP_CREATE_RQ:
1141                 MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1142                 MLX5_SET(destroy_rq_in, din, rqn, *obj_id);
1143                 break;
1144         case MLX5_CMD_OP_CREATE_RQT:
1145                 MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1146                 MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);
1147                 break;
1148         case MLX5_CMD_OP_CREATE_TIR:
1149                 MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1150                 MLX5_SET(destroy_tir_in, din, tirn, *obj_id);
1151                 break;
1152         case MLX5_CMD_OP_CREATE_TIS:
1153                 MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1154                 MLX5_SET(destroy_tis_in, din, tisn, *obj_id);
1155                 break;
1156         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1157                 MLX5_SET(dealloc_q_counter_in, din, opcode,
1158                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1159                 MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);
1160                 break;
1161         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1162                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1163                 MLX5_SET(destroy_flow_table_in, din, other_vport,
1164                          MLX5_GET(create_flow_table_in,  in, other_vport));
1165                 MLX5_SET(destroy_flow_table_in, din, vport_number,
1166                          MLX5_GET(create_flow_table_in,  in, vport_number));
1167                 MLX5_SET(destroy_flow_table_in, din, table_type,
1168                          MLX5_GET(create_flow_table_in,  in, table_type));
1169                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1170                 MLX5_SET(destroy_flow_table_in, din, opcode,
1171                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1172                 break;
1173         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1174                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1175                 MLX5_SET(destroy_flow_group_in, din, other_vport,
1176                          MLX5_GET(create_flow_group_in, in, other_vport));
1177                 MLX5_SET(destroy_flow_group_in, din, vport_number,
1178                          MLX5_GET(create_flow_group_in, in, vport_number));
1179                 MLX5_SET(destroy_flow_group_in, din, table_type,
1180                          MLX5_GET(create_flow_group_in, in, table_type));
1181                 MLX5_SET(destroy_flow_group_in, din, table_id,
1182                          MLX5_GET(create_flow_group_in, in, table_id));
1183                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1184                 MLX5_SET(destroy_flow_group_in, din, opcode,
1185                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1186                 break;
1187         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1188                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1189                 MLX5_SET(delete_fte_in, din, other_vport,
1190                          MLX5_GET(set_fte_in,  in, other_vport));
1191                 MLX5_SET(delete_fte_in, din, vport_number,
1192                          MLX5_GET(set_fte_in, in, vport_number));
1193                 MLX5_SET(delete_fte_in, din, table_type,
1194                          MLX5_GET(set_fte_in, in, table_type));
1195                 MLX5_SET(delete_fte_in, din, table_id,
1196                          MLX5_GET(set_fte_in, in, table_id));
1197                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1198                 MLX5_SET(delete_fte_in, din, opcode,
1199                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1200                 break;
1201         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1202                 MLX5_SET(dealloc_flow_counter_in, din, opcode,
1203                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1204                 MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,
1205                          *obj_id);
1206                 break;
1207         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1208                 MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,
1209                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1210                 MLX5_SET(dealloc_packet_reformat_context_in, din,
1211                          packet_reformat_id, *obj_id);
1212                 break;
1213         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1214                 MLX5_SET(dealloc_modify_header_context_in, din, opcode,
1215                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1216                 MLX5_SET(dealloc_modify_header_context_in, din,
1217                          modify_header_id, *obj_id);
1218                 break;
1219         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1220                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1221                 MLX5_SET(destroy_scheduling_element_in, din,
1222                          scheduling_hierarchy,
1223                          MLX5_GET(create_scheduling_element_in, in,
1224                                   scheduling_hierarchy));
1225                 MLX5_SET(destroy_scheduling_element_in, din,
1226                          scheduling_element_id, *obj_id);
1227                 MLX5_SET(destroy_scheduling_element_in, din, opcode,
1228                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1229                 break;
1230         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1231                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1232                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1233                 MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,
1234                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1235                 break;
1236         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1237                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1238                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1239                 MLX5_SET(delete_l2_table_entry_in, din, opcode,
1240                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1241                 break;
1242         case MLX5_CMD_OP_CREATE_QP:
1243                 MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1244                 MLX5_SET(destroy_qp_in, din, qpn, *obj_id);
1245                 break;
1246         case MLX5_CMD_OP_CREATE_SRQ:
1247                 MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1248                 MLX5_SET(destroy_srq_in, din, srqn, *obj_id);
1249                 break;
1250         case MLX5_CMD_OP_CREATE_XRC_SRQ:
1251                 MLX5_SET(destroy_xrc_srq_in, din, opcode,
1252                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
1253                 MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);
1254                 break;
1255         case MLX5_CMD_OP_CREATE_DCT:
1256                 MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1257                 MLX5_SET(destroy_dct_in, din, dctn, *obj_id);
1258                 break;
1259         case MLX5_CMD_OP_CREATE_XRQ:
1260                 MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1261                 MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);
1262                 break;
1263         case MLX5_CMD_OP_ATTACH_TO_MCG:
1264                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1265                 MLX5_SET(detach_from_mcg_in, din, qpn,
1266                          MLX5_GET(attach_to_mcg_in, in, qpn));
1267                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1268                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1269                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1270                 MLX5_SET(detach_from_mcg_in, din, opcode,
1271                          MLX5_CMD_OP_DETACH_FROM_MCG);
1272                 MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);
1273                 break;
1274         case MLX5_CMD_OP_ALLOC_XRCD:
1275                 MLX5_SET(dealloc_xrcd_in, din, opcode,
1276                          MLX5_CMD_OP_DEALLOC_XRCD);
1277                 MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);
1278                 break;
1279         case MLX5_CMD_OP_CREATE_PSV:
1280                 MLX5_SET(destroy_psv_in, din, opcode,
1281                          MLX5_CMD_OP_DESTROY_PSV);
1282                 MLX5_SET(destroy_psv_in, din, psvn, *obj_id);
1283                 break;
1284         default:
1285                 /* The entry must match to one of the devx_is_obj_create_cmd */
1286                 WARN_ON(true);
1287                 break;
1288         }
1289 }
1290
1291 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1292                                      struct mlx5_ib_dev *dev,
1293                                      void *in, void *out)
1294 {
1295         struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1296         struct mlx5_core_mkey *mkey;
1297         void *mkc;
1298         u8 key;
1299
1300         mkey = &devx_mr->mmkey;
1301         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1302         key = MLX5_GET(mkc, mkc, mkey_7_0);
1303         mkey->key = mlx5_idx_to_mkey(
1304                         MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1305         mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1306         mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1307         mkey->size = MLX5_GET64(mkc, mkc, len);
1308         mkey->pd = MLX5_GET(mkc, mkc, pd);
1309         devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1310         init_waitqueue_head(&mkey->wait);
1311
1312         return mlx5r_store_odp_mkey(dev, mkey);
1313 }
1314
1315 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1316                                    struct devx_obj *obj,
1317                                    void *in, int in_len)
1318 {
1319         int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1320                         MLX5_FLD_SZ_BYTES(create_mkey_in,
1321                         memory_key_mkey_entry);
1322         void *mkc;
1323         u8 access_mode;
1324
1325         if (in_len < min_len)
1326                 return -EINVAL;
1327
1328         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1329
1330         access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1331         access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1332
1333         if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1334                 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1335                 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1336                         obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1337                 return 0;
1338         }
1339
1340         MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1341         return 0;
1342 }
1343
1344 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1345                                       struct devx_event_subscription *sub)
1346 {
1347         struct devx_event *event;
1348         struct devx_obj_event *xa_val_level2;
1349
1350         if (sub->is_cleaned)
1351                 return;
1352
1353         sub->is_cleaned = 1;
1354         list_del_rcu(&sub->xa_list);
1355
1356         if (list_empty(&sub->obj_list))
1357                 return;
1358
1359         list_del_rcu(&sub->obj_list);
1360         /* check whether key level 1 for this obj_sub_list is empty */
1361         event = xa_load(&dev->devx_event_table.event_xa,
1362                         sub->xa_key_level1);
1363         WARN_ON(!event);
1364
1365         xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1366         if (list_empty(&xa_val_level2->obj_sub_list)) {
1367                 xa_erase(&event->object_ids,
1368                          sub->xa_key_level2);
1369                 kfree_rcu(xa_val_level2, rcu);
1370         }
1371 }
1372
1373 static int devx_obj_cleanup(struct ib_uobject *uobject,
1374                             enum rdma_remove_reason why,
1375                             struct uverbs_attr_bundle *attrs)
1376 {
1377         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1378         struct mlx5_devx_event_table *devx_event_table;
1379         struct devx_obj *obj = uobject->object;
1380         struct devx_event_subscription *sub_entry, *tmp;
1381         struct mlx5_ib_dev *dev;
1382         int ret;
1383
1384         dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1385         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
1386             xa_erase(&obj->ib_dev->odp_mkeys,
1387                      mlx5_base_mkey(obj->devx_mr.mmkey.key)))
1388                 /*
1389                  * The pagefault_single_data_segment() does commands against
1390                  * the mmkey, we must wait for that to stop before freeing the
1391                  * mkey, as another allocation could get the same mkey #.
1392                  */
1393                 mlx5r_deref_wait_odp_mkey(&obj->devx_mr.mmkey);
1394
1395         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1396                 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1397         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1398                 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1399         else
1400                 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1401                                     obj->dinlen, out, sizeof(out));
1402         if (ret)
1403                 return ret;
1404
1405         devx_event_table = &dev->devx_event_table;
1406
1407         mutex_lock(&devx_event_table->event_xa_lock);
1408         list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1409                 devx_cleanup_subscription(dev, sub_entry);
1410         mutex_unlock(&devx_event_table->event_xa_lock);
1411
1412         kfree(obj);
1413         return ret;
1414 }
1415
1416 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1417 {
1418         struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1419         struct mlx5_devx_event_table *table;
1420         struct devx_event *event;
1421         struct devx_obj_event *obj_event;
1422         u32 obj_id = mcq->cqn;
1423
1424         table = &obj->ib_dev->devx_event_table;
1425         rcu_read_lock();
1426         event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1427         if (!event)
1428                 goto out;
1429
1430         obj_event = xa_load(&event->object_ids, obj_id);
1431         if (!obj_event)
1432                 goto out;
1433
1434         dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1435 out:
1436         rcu_read_unlock();
1437 }
1438
1439 static bool is_apu_thread_cq(struct mlx5_ib_dev *dev, const void *in)
1440 {
1441         if (!MLX5_CAP_GEN(dev->mdev, apu) ||
1442             !MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context),
1443                       apu_thread_cq))
1444                 return false;
1445
1446         return true;
1447 }
1448
1449 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1450         struct uverbs_attr_bundle *attrs)
1451 {
1452         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1453         int cmd_out_len =  uverbs_attr_get_len(attrs,
1454                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1455         int cmd_in_len = uverbs_attr_get_len(attrs,
1456                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1457         void *cmd_out;
1458         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1459                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1460         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1461                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1462         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1463         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1464         struct devx_obj *obj;
1465         u16 obj_type = 0;
1466         int err;
1467         int uid;
1468         u32 obj_id;
1469         u16 opcode;
1470
1471         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1472                 return -EINVAL;
1473
1474         uid = devx_get_uid(c, cmd_in);
1475         if (uid < 0)
1476                 return uid;
1477
1478         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1479                 return -EINVAL;
1480
1481         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1482         if (IS_ERR(cmd_out))
1483                 return PTR_ERR(cmd_out);
1484
1485         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1486         if (!obj)
1487                 return -ENOMEM;
1488
1489         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1490         if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1491                 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1492                 if (err)
1493                         goto obj_free;
1494         } else {
1495                 devx_set_umem_valid(cmd_in);
1496         }
1497
1498         if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1499                 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1500                 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1501                                            cmd_in_len, cmd_out, cmd_out_len);
1502         } else if (opcode == MLX5_CMD_OP_CREATE_CQ &&
1503                    !is_apu_thread_cq(dev, cmd_in)) {
1504                 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1505                 obj->core_cq.comp = devx_cq_comp;
1506                 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1507                                           cmd_in, cmd_in_len, cmd_out,
1508                                           cmd_out_len);
1509         } else {
1510                 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1511                                     cmd_in_len,
1512                                     cmd_out, cmd_out_len);
1513         }
1514
1515         if (err)
1516                 goto obj_free;
1517
1518         if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1519                 u8 bulk = MLX5_GET(alloc_flow_counter_in,
1520                                    cmd_in,
1521                                    flow_counter_bulk);
1522                 obj->flow_counter_bulk_size = 128UL * bulk;
1523         }
1524
1525         uobj->object = obj;
1526         INIT_LIST_HEAD(&obj->event_sub);
1527         obj->ib_dev = dev;
1528         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1529                                    &obj_id);
1530         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1531
1532         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1533         if (err)
1534                 goto obj_destroy;
1535
1536         if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1537                 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1538         obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1539
1540         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1541                 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1542                 if (err)
1543                         goto obj_destroy;
1544         }
1545         return 0;
1546
1547 obj_destroy:
1548         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1549                 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1550         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1551                 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1552         else
1553                 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1554                               sizeof(out));
1555 obj_free:
1556         kfree(obj);
1557         return err;
1558 }
1559
1560 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1561         struct uverbs_attr_bundle *attrs)
1562 {
1563         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1564         int cmd_out_len = uverbs_attr_get_len(attrs,
1565                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1566         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1567                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1568         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1569                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1570         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1571         void *cmd_out;
1572         int err;
1573         int uid;
1574
1575         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1576                 return -EINVAL;
1577
1578         uid = devx_get_uid(c, cmd_in);
1579         if (uid < 0)
1580                 return uid;
1581
1582         if (!devx_is_obj_modify_cmd(cmd_in))
1583                 return -EINVAL;
1584
1585         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1586                 return -EINVAL;
1587
1588         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1589         if (IS_ERR(cmd_out))
1590                 return PTR_ERR(cmd_out);
1591
1592         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1593         devx_set_umem_valid(cmd_in);
1594
1595         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1596                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1597                             cmd_out, cmd_out_len);
1598         if (err)
1599                 return err;
1600
1601         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1602                               cmd_out, cmd_out_len);
1603 }
1604
1605 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1606         struct uverbs_attr_bundle *attrs)
1607 {
1608         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1609         int cmd_out_len = uverbs_attr_get_len(attrs,
1610                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1611         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1612                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1613         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1614                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1615         void *cmd_out;
1616         int err;
1617         int uid;
1618         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1619
1620         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1621                 return -EINVAL;
1622
1623         uid = devx_get_uid(c, cmd_in);
1624         if (uid < 0)
1625                 return uid;
1626
1627         if (!devx_is_obj_query_cmd(cmd_in))
1628                 return -EINVAL;
1629
1630         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1631                 return -EINVAL;
1632
1633         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1634         if (IS_ERR(cmd_out))
1635                 return PTR_ERR(cmd_out);
1636
1637         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1638         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1639                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1640                             cmd_out, cmd_out_len);
1641         if (err)
1642                 return err;
1643
1644         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1645                               cmd_out, cmd_out_len);
1646 }
1647
1648 struct devx_async_event_queue {
1649         spinlock_t              lock;
1650         wait_queue_head_t       poll_wait;
1651         struct list_head        event_list;
1652         atomic_t                bytes_in_use;
1653         u8                      is_destroyed:1;
1654 };
1655
1656 struct devx_async_cmd_event_file {
1657         struct ib_uobject               uobj;
1658         struct devx_async_event_queue   ev_queue;
1659         struct mlx5_async_ctx           async_ctx;
1660 };
1661
1662 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1663 {
1664         spin_lock_init(&ev_queue->lock);
1665         INIT_LIST_HEAD(&ev_queue->event_list);
1666         init_waitqueue_head(&ev_queue->poll_wait);
1667         atomic_set(&ev_queue->bytes_in_use, 0);
1668         ev_queue->is_destroyed = 0;
1669 }
1670
1671 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1672         struct uverbs_attr_bundle *attrs)
1673 {
1674         struct devx_async_cmd_event_file *ev_file;
1675
1676         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1677                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1678         struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1679
1680         ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1681                                uobj);
1682         devx_init_event_queue(&ev_file->ev_queue);
1683         mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1684         return 0;
1685 }
1686
1687 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1688         struct uverbs_attr_bundle *attrs)
1689 {
1690         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1691                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1692         struct devx_async_event_file *ev_file;
1693         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1694                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1695         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1696         u32 flags;
1697         int err;
1698
1699         err = uverbs_get_flags32(&flags, attrs,
1700                 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1701                 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1702
1703         if (err)
1704                 return err;
1705
1706         ev_file = container_of(uobj, struct devx_async_event_file,
1707                                uobj);
1708         spin_lock_init(&ev_file->lock);
1709         INIT_LIST_HEAD(&ev_file->event_list);
1710         init_waitqueue_head(&ev_file->poll_wait);
1711         if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1712                 ev_file->omit_data = 1;
1713         INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1714         ev_file->dev = dev;
1715         get_device(&dev->ib_dev.dev);
1716         return 0;
1717 }
1718
1719 static void devx_query_callback(int status, struct mlx5_async_work *context)
1720 {
1721         struct devx_async_data *async_data =
1722                 container_of(context, struct devx_async_data, cb_work);
1723         struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1724         struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1725         unsigned long flags;
1726
1727         /*
1728          * Note that if the struct devx_async_cmd_event_file uobj begins to be
1729          * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1730          * routine returns, ensuring that it always remains valid here.
1731          */
1732         spin_lock_irqsave(&ev_queue->lock, flags);
1733         list_add_tail(&async_data->list, &ev_queue->event_list);
1734         spin_unlock_irqrestore(&ev_queue->lock, flags);
1735
1736         wake_up_interruptible(&ev_queue->poll_wait);
1737 }
1738
1739 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1740
1741 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1742         struct uverbs_attr_bundle *attrs)
1743 {
1744         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1745                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1746         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1747                                 attrs,
1748                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1749         u16 cmd_out_len;
1750         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1751                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1752         struct ib_uobject *fd_uobj;
1753         int err;
1754         int uid;
1755         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1756         struct devx_async_cmd_event_file *ev_file;
1757         struct devx_async_data *async_data;
1758
1759         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1760                 return -EINVAL;
1761
1762         uid = devx_get_uid(c, cmd_in);
1763         if (uid < 0)
1764                 return uid;
1765
1766         if (!devx_is_obj_query_cmd(cmd_in))
1767                 return -EINVAL;
1768
1769         err = uverbs_get_const(&cmd_out_len, attrs,
1770                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1771         if (err)
1772                 return err;
1773
1774         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1775                 return -EINVAL;
1776
1777         fd_uobj = uverbs_attr_get_uobject(attrs,
1778                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1779         if (IS_ERR(fd_uobj))
1780                 return PTR_ERR(fd_uobj);
1781
1782         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1783                                uobj);
1784
1785         if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1786                         MAX_ASYNC_BYTES_IN_USE) {
1787                 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1788                 return -EAGAIN;
1789         }
1790
1791         async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1792                                           cmd_out_len), GFP_KERNEL);
1793         if (!async_data) {
1794                 err = -ENOMEM;
1795                 goto sub_bytes;
1796         }
1797
1798         err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1799                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1800         if (err)
1801                 goto free_async;
1802
1803         async_data->cmd_out_len = cmd_out_len;
1804         async_data->mdev = mdev;
1805         async_data->ev_file = ev_file;
1806
1807         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1808         err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1809                     uverbs_attr_get_len(attrs,
1810                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1811                     async_data->hdr.out_data,
1812                     async_data->cmd_out_len,
1813                     devx_query_callback, &async_data->cb_work);
1814
1815         if (err)
1816                 goto free_async;
1817
1818         return 0;
1819
1820 free_async:
1821         kvfree(async_data);
1822 sub_bytes:
1823         atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1824         return err;
1825 }
1826
1827 static void
1828 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1829                            u32 key_level1,
1830                            bool is_level2,
1831                            u32 key_level2)
1832 {
1833         struct devx_event *event;
1834         struct devx_obj_event *xa_val_level2;
1835
1836         /* Level 1 is valid for future use, no need to free */
1837         if (!is_level2)
1838                 return;
1839
1840         event = xa_load(&devx_event_table->event_xa, key_level1);
1841         WARN_ON(!event);
1842
1843         xa_val_level2 = xa_load(&event->object_ids,
1844                                 key_level2);
1845         if (list_empty(&xa_val_level2->obj_sub_list)) {
1846                 xa_erase(&event->object_ids,
1847                          key_level2);
1848                 kfree_rcu(xa_val_level2, rcu);
1849         }
1850 }
1851
1852 static int
1853 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1854                          u32 key_level1,
1855                          bool is_level2,
1856                          u32 key_level2)
1857 {
1858         struct devx_obj_event *obj_event;
1859         struct devx_event *event;
1860         int err;
1861
1862         event = xa_load(&devx_event_table->event_xa, key_level1);
1863         if (!event) {
1864                 event = kzalloc(sizeof(*event), GFP_KERNEL);
1865                 if (!event)
1866                         return -ENOMEM;
1867
1868                 INIT_LIST_HEAD(&event->unaffiliated_list);
1869                 xa_init(&event->object_ids);
1870
1871                 err = xa_insert(&devx_event_table->event_xa,
1872                                 key_level1,
1873                                 event,
1874                                 GFP_KERNEL);
1875                 if (err) {
1876                         kfree(event);
1877                         return err;
1878                 }
1879         }
1880
1881         if (!is_level2)
1882                 return 0;
1883
1884         obj_event = xa_load(&event->object_ids, key_level2);
1885         if (!obj_event) {
1886                 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1887                 if (!obj_event)
1888                         /* Level1 is valid for future use, no need to free */
1889                         return -ENOMEM;
1890
1891                 err = xa_insert(&event->object_ids,
1892                                 key_level2,
1893                                 obj_event,
1894                                 GFP_KERNEL);
1895                 if (err)
1896                         return err;
1897                 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1898         }
1899
1900         return 0;
1901 }
1902
1903 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1904                                    struct devx_obj *obj)
1905 {
1906         int i;
1907
1908         for (i = 0; i < num_events; i++) {
1909                 if (obj) {
1910                         if (!is_legacy_obj_event_num(event_type_num_list[i]))
1911                                 return false;
1912                 } else if (!is_legacy_unaffiliated_event_num(
1913                                 event_type_num_list[i])) {
1914                         return false;
1915                 }
1916         }
1917
1918         return true;
1919 }
1920
1921 #define MAX_SUPP_EVENT_NUM 255
1922 static bool is_valid_events(struct mlx5_core_dev *dev,
1923                             int num_events, u16 *event_type_num_list,
1924                             struct devx_obj *obj)
1925 {
1926         __be64 *aff_events;
1927         __be64 *unaff_events;
1928         int mask_entry;
1929         int mask_bit;
1930         int i;
1931
1932         if (MLX5_CAP_GEN(dev, event_cap)) {
1933                 aff_events = MLX5_CAP_DEV_EVENT(dev,
1934                                                 user_affiliated_events);
1935                 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1936                                                   user_unaffiliated_events);
1937         } else {
1938                 return is_valid_events_legacy(num_events, event_type_num_list,
1939                                               obj);
1940         }
1941
1942         for (i = 0; i < num_events; i++) {
1943                 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1944                         return false;
1945
1946                 mask_entry = event_type_num_list[i] / 64;
1947                 mask_bit = event_type_num_list[i] % 64;
1948
1949                 if (obj) {
1950                         /* CQ completion */
1951                         if (event_type_num_list[i] == 0)
1952                                 continue;
1953
1954                         if (!(be64_to_cpu(aff_events[mask_entry]) &
1955                                         (1ull << mask_bit)))
1956                                 return false;
1957
1958                         continue;
1959                 }
1960
1961                 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1962                                 (1ull << mask_bit)))
1963                         return false;
1964         }
1965
1966         return true;
1967 }
1968
1969 #define MAX_NUM_EVENTS 16
1970 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1971         struct uverbs_attr_bundle *attrs)
1972 {
1973         struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1974                                 attrs,
1975                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1976         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1977                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1978         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1979         struct ib_uobject *fd_uobj;
1980         struct devx_obj *obj = NULL;
1981         struct devx_async_event_file *ev_file;
1982         struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1983         u16 *event_type_num_list;
1984         struct devx_event_subscription *event_sub, *tmp_sub;
1985         struct list_head sub_list;
1986         int redirect_fd;
1987         bool use_eventfd = false;
1988         int num_events;
1989         int num_alloc_xa_entries = 0;
1990         u16 obj_type = 0;
1991         u64 cookie = 0;
1992         u32 obj_id = 0;
1993         int err;
1994         int i;
1995
1996         if (!c->devx_uid)
1997                 return -EINVAL;
1998
1999         if (!IS_ERR(devx_uobj)) {
2000                 obj = (struct devx_obj *)devx_uobj->object;
2001                 if (obj)
2002                         obj_id = get_dec_obj_id(obj->obj_id);
2003         }
2004
2005         fd_uobj = uverbs_attr_get_uobject(attrs,
2006                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
2007         if (IS_ERR(fd_uobj))
2008                 return PTR_ERR(fd_uobj);
2009
2010         ev_file = container_of(fd_uobj, struct devx_async_event_file,
2011                                uobj);
2012
2013         if (uverbs_attr_is_valid(attrs,
2014                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
2015                 err = uverbs_copy_from(&redirect_fd, attrs,
2016                                MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
2017                 if (err)
2018                         return err;
2019
2020                 use_eventfd = true;
2021         }
2022
2023         if (uverbs_attr_is_valid(attrs,
2024                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
2025                 if (use_eventfd)
2026                         return -EINVAL;
2027
2028                 err = uverbs_copy_from(&cookie, attrs,
2029                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
2030                 if (err)
2031                         return err;
2032         }
2033
2034         num_events = uverbs_attr_ptr_get_array_size(
2035                 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2036                 sizeof(u16));
2037
2038         if (num_events < 0)
2039                 return num_events;
2040
2041         if (num_events > MAX_NUM_EVENTS)
2042                 return -EINVAL;
2043
2044         event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2045                         MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2046
2047         if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2048                 return -EINVAL;
2049
2050         INIT_LIST_HEAD(&sub_list);
2051
2052         /* Protect from concurrent subscriptions to same XA entries to allow
2053          * both to succeed
2054          */
2055         mutex_lock(&devx_event_table->event_xa_lock);
2056         for (i = 0; i < num_events; i++) {
2057                 u32 key_level1;
2058
2059                 if (obj)
2060                         obj_type = get_dec_obj_type(obj,
2061                                                     event_type_num_list[i]);
2062                 key_level1 = event_type_num_list[i] | obj_type << 16;
2063
2064                 err = subscribe_event_xa_alloc(devx_event_table,
2065                                                key_level1,
2066                                                obj,
2067                                                obj_id);
2068                 if (err)
2069                         goto err;
2070
2071                 num_alloc_xa_entries++;
2072                 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2073                 if (!event_sub) {
2074                         err = -ENOMEM;
2075                         goto err;
2076                 }
2077
2078                 list_add_tail(&event_sub->event_list, &sub_list);
2079                 uverbs_uobject_get(&ev_file->uobj);
2080                 if (use_eventfd) {
2081                         event_sub->eventfd =
2082                                 eventfd_ctx_fdget(redirect_fd);
2083
2084                         if (IS_ERR(event_sub->eventfd)) {
2085                                 err = PTR_ERR(event_sub->eventfd);
2086                                 event_sub->eventfd = NULL;
2087                                 goto err;
2088                         }
2089                 }
2090
2091                 event_sub->cookie = cookie;
2092                 event_sub->ev_file = ev_file;
2093                 /* May be needed upon cleanup the devx object/subscription */
2094                 event_sub->xa_key_level1 = key_level1;
2095                 event_sub->xa_key_level2 = obj_id;
2096                 INIT_LIST_HEAD(&event_sub->obj_list);
2097         }
2098
2099         /* Once all the allocations and the XA data insertions were done we
2100          * can go ahead and add all the subscriptions to the relevant lists
2101          * without concern of a failure.
2102          */
2103         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2104                 struct devx_event *event;
2105                 struct devx_obj_event *obj_event;
2106
2107                 list_del_init(&event_sub->event_list);
2108
2109                 spin_lock_irq(&ev_file->lock);
2110                 list_add_tail_rcu(&event_sub->file_list,
2111                                   &ev_file->subscribed_events_list);
2112                 spin_unlock_irq(&ev_file->lock);
2113
2114                 event = xa_load(&devx_event_table->event_xa,
2115                                 event_sub->xa_key_level1);
2116                 WARN_ON(!event);
2117
2118                 if (!obj) {
2119                         list_add_tail_rcu(&event_sub->xa_list,
2120                                           &event->unaffiliated_list);
2121                         continue;
2122                 }
2123
2124                 obj_event = xa_load(&event->object_ids, obj_id);
2125                 WARN_ON(!obj_event);
2126                 list_add_tail_rcu(&event_sub->xa_list,
2127                                   &obj_event->obj_sub_list);
2128                 list_add_tail_rcu(&event_sub->obj_list,
2129                                   &obj->event_sub);
2130         }
2131
2132         mutex_unlock(&devx_event_table->event_xa_lock);
2133         return 0;
2134
2135 err:
2136         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2137                 list_del(&event_sub->event_list);
2138
2139                 subscribe_event_xa_dealloc(devx_event_table,
2140                                            event_sub->xa_key_level1,
2141                                            obj,
2142                                            obj_id);
2143
2144                 if (event_sub->eventfd)
2145                         eventfd_ctx_put(event_sub->eventfd);
2146                 uverbs_uobject_put(&event_sub->ev_file->uobj);
2147                 kfree(event_sub);
2148         }
2149
2150         mutex_unlock(&devx_event_table->event_xa_lock);
2151         return err;
2152 }
2153
2154 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2155                          struct uverbs_attr_bundle *attrs,
2156                          struct devx_umem *obj)
2157 {
2158         u64 addr;
2159         size_t size;
2160         u32 access;
2161         int err;
2162
2163         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2164             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2165                 return -EFAULT;
2166
2167         err = uverbs_get_flags32(&access, attrs,
2168                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2169                                  IB_ACCESS_LOCAL_WRITE |
2170                                  IB_ACCESS_REMOTE_WRITE |
2171                                  IB_ACCESS_REMOTE_READ);
2172         if (err)
2173                 return err;
2174
2175         err = ib_check_mr_access(&dev->ib_dev, access);
2176         if (err)
2177                 return err;
2178
2179         obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
2180         if (IS_ERR(obj->umem))
2181                 return PTR_ERR(obj->umem);
2182         return 0;
2183 }
2184
2185 static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,
2186                                                unsigned long pgsz_bitmap)
2187 {
2188         unsigned long page_size;
2189
2190         /* Don't bother checking larger page sizes as offset must be zero and
2191          * total DEVX umem length must be equal to total umem length.
2192          */
2193         pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
2194                                          PAGE_SHIFT),
2195                                    MLX5_ADAPTER_PAGE_SHIFT);
2196         if (!pgsz_bitmap)
2197                 return 0;
2198
2199         page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);
2200         if (!page_size)
2201                 return 0;
2202
2203         /* If the page_size is less than the CPU page size then we can use the
2204          * offset and create a umem which is a subset of the page list.
2205          * For larger page sizes we can't be sure the DMA  list reflects the
2206          * VA so we must ensure that the umem extent is exactly equal to the
2207          * page list. Reduce the page size until one of these cases is true.
2208          */
2209         while ((ib_umem_dma_offset(umem, page_size) != 0 ||
2210                 (umem->length % page_size) != 0) &&
2211                 page_size > PAGE_SIZE)
2212                 page_size /= 2;
2213
2214         return page_size;
2215 }
2216
2217 static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2218                                    struct uverbs_attr_bundle *attrs,
2219                                    struct devx_umem *obj,
2220                                    struct devx_umem_reg_cmd *cmd)
2221 {
2222         unsigned long pgsz_bitmap;
2223         unsigned int page_size;
2224         __be64 *mtt;
2225         void *umem;
2226         int ret;
2227
2228         /*
2229          * If the user does not pass in pgsz_bitmap then the user promises not
2230          * to use umem_offset!=0 in any commands that allocate on top of the
2231          * umem.
2232          *
2233          * If the user wants to use a umem_offset then it must pass in
2234          * pgsz_bitmap which guides the maximum page size and thus maximum
2235          * object alignment inside the umem. See the PRM.
2236          *
2237          * Users are not allowed to use IOVA here, mkeys are not supported on
2238          * umem.
2239          */
2240         ret = uverbs_get_const_default(&pgsz_bitmap, attrs,
2241                         MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2242                         GENMASK_ULL(63,
2243                                     min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));
2244         if (ret)
2245                 return ret;
2246
2247         page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap);
2248         if (!page_size)
2249                 return -EINVAL;
2250
2251         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2252                      (MLX5_ST_SZ_BYTES(mtt) *
2253                       ib_umem_num_dma_blocks(obj->umem, page_size));
2254         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2255         if (IS_ERR(cmd->in))
2256                 return PTR_ERR(cmd->in);
2257
2258         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2259         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2260
2261         MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2262         MLX5_SET64(umem, umem, num_of_mtt,
2263                    ib_umem_num_dma_blocks(obj->umem, page_size));
2264         MLX5_SET(umem, umem, log_page_size,
2265                  order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2266         MLX5_SET(umem, umem, page_offset,
2267                  ib_umem_dma_offset(obj->umem, page_size));
2268
2269         mlx5_ib_populate_pas(obj->umem, page_size, mtt,
2270                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2271                                      MLX5_IB_MTT_READ);
2272         return 0;
2273 }
2274
2275 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2276         struct uverbs_attr_bundle *attrs)
2277 {
2278         struct devx_umem_reg_cmd cmd;
2279         struct devx_umem *obj;
2280         struct ib_uobject *uobj = uverbs_attr_get_uobject(
2281                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2282         u32 obj_id;
2283         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2284                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2285         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2286         int err;
2287
2288         if (!c->devx_uid)
2289                 return -EINVAL;
2290
2291         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2292         if (!obj)
2293                 return -ENOMEM;
2294
2295         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2296         if (err)
2297                 goto err_obj_free;
2298
2299         err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd);
2300         if (err)
2301                 goto err_umem_release;
2302
2303         MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2304         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2305                             sizeof(cmd.out));
2306         if (err)
2307                 goto err_umem_release;
2308
2309         obj->mdev = dev->mdev;
2310         uobj->object = obj;
2311         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2312         uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2313
2314         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2315                              sizeof(obj_id));
2316         return err;
2317
2318 err_umem_release:
2319         ib_umem_release(obj->umem);
2320 err_obj_free:
2321         kfree(obj);
2322         return err;
2323 }
2324
2325 static int devx_umem_cleanup(struct ib_uobject *uobject,
2326                              enum rdma_remove_reason why,
2327                              struct uverbs_attr_bundle *attrs)
2328 {
2329         struct devx_umem *obj = uobject->object;
2330         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2331         int err;
2332
2333         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2334         if (err)
2335                 return err;
2336
2337         ib_umem_release(obj->umem);
2338         kfree(obj);
2339         return 0;
2340 }
2341
2342 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2343                                   unsigned long event_type)
2344 {
2345         __be64 *unaff_events;
2346         int mask_entry;
2347         int mask_bit;
2348
2349         if (!MLX5_CAP_GEN(dev, event_cap))
2350                 return is_legacy_unaffiliated_event_num(event_type);
2351
2352         unaff_events = MLX5_CAP_DEV_EVENT(dev,
2353                                           user_unaffiliated_events);
2354         WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2355
2356         mask_entry = event_type / 64;
2357         mask_bit = event_type % 64;
2358
2359         if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2360                 return false;
2361
2362         return true;
2363 }
2364
2365 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2366 {
2367         struct mlx5_eqe *eqe = data;
2368         u32 obj_id = 0;
2369
2370         switch (event_type) {
2371         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2372         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2373         case MLX5_EVENT_TYPE_PATH_MIG:
2374         case MLX5_EVENT_TYPE_COMM_EST:
2375         case MLX5_EVENT_TYPE_SQ_DRAINED:
2376         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2377         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2378         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2379         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2380         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2381                 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2382                 break;
2383         case MLX5_EVENT_TYPE_XRQ_ERROR:
2384                 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2385                 break;
2386         case MLX5_EVENT_TYPE_DCT_DRAINED:
2387         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2388                 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2389                 break;
2390         case MLX5_EVENT_TYPE_CQ_ERROR:
2391                 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2392                 break;
2393         default:
2394                 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2395                 break;
2396         }
2397
2398         return obj_id;
2399 }
2400
2401 static int deliver_event(struct devx_event_subscription *event_sub,
2402                          const void *data)
2403 {
2404         struct devx_async_event_file *ev_file;
2405         struct devx_async_event_data *event_data;
2406         unsigned long flags;
2407
2408         ev_file = event_sub->ev_file;
2409
2410         if (ev_file->omit_data) {
2411                 spin_lock_irqsave(&ev_file->lock, flags);
2412                 if (!list_empty(&event_sub->event_list) ||
2413                     ev_file->is_destroyed) {
2414                         spin_unlock_irqrestore(&ev_file->lock, flags);
2415                         return 0;
2416                 }
2417
2418                 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2419                 spin_unlock_irqrestore(&ev_file->lock, flags);
2420                 wake_up_interruptible(&ev_file->poll_wait);
2421                 return 0;
2422         }
2423
2424         event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2425                              GFP_ATOMIC);
2426         if (!event_data) {
2427                 spin_lock_irqsave(&ev_file->lock, flags);
2428                 ev_file->is_overflow_err = 1;
2429                 spin_unlock_irqrestore(&ev_file->lock, flags);
2430                 return -ENOMEM;
2431         }
2432
2433         event_data->hdr.cookie = event_sub->cookie;
2434         memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2435
2436         spin_lock_irqsave(&ev_file->lock, flags);
2437         if (!ev_file->is_destroyed)
2438                 list_add_tail(&event_data->list, &ev_file->event_list);
2439         else
2440                 kfree(event_data);
2441         spin_unlock_irqrestore(&ev_file->lock, flags);
2442         wake_up_interruptible(&ev_file->poll_wait);
2443
2444         return 0;
2445 }
2446
2447 static void dispatch_event_fd(struct list_head *fd_list,
2448                               const void *data)
2449 {
2450         struct devx_event_subscription *item;
2451
2452         list_for_each_entry_rcu(item, fd_list, xa_list) {
2453                 if (item->eventfd)
2454                         eventfd_signal(item->eventfd, 1);
2455                 else
2456                         deliver_event(item, data);
2457         }
2458 }
2459
2460 static int devx_event_notifier(struct notifier_block *nb,
2461                                unsigned long event_type, void *data)
2462 {
2463         struct mlx5_devx_event_table *table;
2464         struct mlx5_ib_dev *dev;
2465         struct devx_event *event;
2466         struct devx_obj_event *obj_event;
2467         u16 obj_type = 0;
2468         bool is_unaffiliated;
2469         u32 obj_id;
2470
2471         /* Explicit filtering to kernel events which may occur frequently */
2472         if (event_type == MLX5_EVENT_TYPE_CMD ||
2473             event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2474                 return NOTIFY_OK;
2475
2476         table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2477         dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2478         is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2479
2480         if (!is_unaffiliated)
2481                 obj_type = get_event_obj_type(event_type, data);
2482
2483         rcu_read_lock();
2484         event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2485         if (!event) {
2486                 rcu_read_unlock();
2487                 return NOTIFY_DONE;
2488         }
2489
2490         if (is_unaffiliated) {
2491                 dispatch_event_fd(&event->unaffiliated_list, data);
2492                 rcu_read_unlock();
2493                 return NOTIFY_OK;
2494         }
2495
2496         obj_id = devx_get_obj_id_from_event(event_type, data);
2497         obj_event = xa_load(&event->object_ids, obj_id);
2498         if (!obj_event) {
2499                 rcu_read_unlock();
2500                 return NOTIFY_DONE;
2501         }
2502
2503         dispatch_event_fd(&obj_event->obj_sub_list, data);
2504
2505         rcu_read_unlock();
2506         return NOTIFY_OK;
2507 }
2508
2509 int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
2510 {
2511         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2512         int uid;
2513
2514         uid = mlx5_ib_devx_create(dev, false);
2515         if (uid > 0) {
2516                 dev->devx_whitelist_uid = uid;
2517                 xa_init(&table->event_xa);
2518                 mutex_init(&table->event_xa_lock);
2519                 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2520                 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2521         }
2522
2523         return 0;
2524 }
2525
2526 void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
2527 {
2528         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2529         struct devx_event_subscription *sub, *tmp;
2530         struct devx_event *event;
2531         void *entry;
2532         unsigned long id;
2533
2534         if (dev->devx_whitelist_uid) {
2535                 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2536                 mutex_lock(&dev->devx_event_table.event_xa_lock);
2537                 xa_for_each(&table->event_xa, id, entry) {
2538                         event = entry;
2539                         list_for_each_entry_safe(
2540                                 sub, tmp, &event->unaffiliated_list, xa_list)
2541                                 devx_cleanup_subscription(dev, sub);
2542                         kfree(entry);
2543                 }
2544                 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2545                 xa_destroy(&table->event_xa);
2546
2547                 mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
2548         }
2549 }
2550
2551 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2552                                          size_t count, loff_t *pos)
2553 {
2554         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2555         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2556         struct devx_async_data *event;
2557         int ret = 0;
2558         size_t eventsz;
2559
2560         spin_lock_irq(&ev_queue->lock);
2561
2562         while (list_empty(&ev_queue->event_list)) {
2563                 spin_unlock_irq(&ev_queue->lock);
2564
2565                 if (filp->f_flags & O_NONBLOCK)
2566                         return -EAGAIN;
2567
2568                 if (wait_event_interruptible(
2569                             ev_queue->poll_wait,
2570                             (!list_empty(&ev_queue->event_list) ||
2571                              ev_queue->is_destroyed))) {
2572                         return -ERESTARTSYS;
2573                 }
2574
2575                 spin_lock_irq(&ev_queue->lock);
2576                 if (ev_queue->is_destroyed) {
2577                         spin_unlock_irq(&ev_queue->lock);
2578                         return -EIO;
2579                 }
2580         }
2581
2582         event = list_entry(ev_queue->event_list.next,
2583                            struct devx_async_data, list);
2584         eventsz = event->cmd_out_len +
2585                         sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2586
2587         if (eventsz > count) {
2588                 spin_unlock_irq(&ev_queue->lock);
2589                 return -ENOSPC;
2590         }
2591
2592         list_del(ev_queue->event_list.next);
2593         spin_unlock_irq(&ev_queue->lock);
2594
2595         if (copy_to_user(buf, &event->hdr, eventsz))
2596                 ret = -EFAULT;
2597         else
2598                 ret = eventsz;
2599
2600         atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2601         kvfree(event);
2602         return ret;
2603 }
2604
2605 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2606                                               struct poll_table_struct *wait)
2607 {
2608         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2609         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2610         __poll_t pollflags = 0;
2611
2612         poll_wait(filp, &ev_queue->poll_wait, wait);
2613
2614         spin_lock_irq(&ev_queue->lock);
2615         if (ev_queue->is_destroyed)
2616                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2617         else if (!list_empty(&ev_queue->event_list))
2618                 pollflags = EPOLLIN | EPOLLRDNORM;
2619         spin_unlock_irq(&ev_queue->lock);
2620
2621         return pollflags;
2622 }
2623
2624 static const struct file_operations devx_async_cmd_event_fops = {
2625         .owner   = THIS_MODULE,
2626         .read    = devx_async_cmd_event_read,
2627         .poll    = devx_async_cmd_event_poll,
2628         .release = uverbs_uobject_fd_release,
2629         .llseek  = no_llseek,
2630 };
2631
2632 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2633                                      size_t count, loff_t *pos)
2634 {
2635         struct devx_async_event_file *ev_file = filp->private_data;
2636         struct devx_event_subscription *event_sub;
2637         struct devx_async_event_data *event;
2638         int ret = 0;
2639         size_t eventsz;
2640         bool omit_data;
2641         void *event_data;
2642
2643         omit_data = ev_file->omit_data;
2644
2645         spin_lock_irq(&ev_file->lock);
2646
2647         if (ev_file->is_overflow_err) {
2648                 ev_file->is_overflow_err = 0;
2649                 spin_unlock_irq(&ev_file->lock);
2650                 return -EOVERFLOW;
2651         }
2652
2653
2654         while (list_empty(&ev_file->event_list)) {
2655                 spin_unlock_irq(&ev_file->lock);
2656
2657                 if (filp->f_flags & O_NONBLOCK)
2658                         return -EAGAIN;
2659
2660                 if (wait_event_interruptible(ev_file->poll_wait,
2661                             (!list_empty(&ev_file->event_list) ||
2662                              ev_file->is_destroyed))) {
2663                         return -ERESTARTSYS;
2664                 }
2665
2666                 spin_lock_irq(&ev_file->lock);
2667                 if (ev_file->is_destroyed) {
2668                         spin_unlock_irq(&ev_file->lock);
2669                         return -EIO;
2670                 }
2671         }
2672
2673         if (omit_data) {
2674                 event_sub = list_first_entry(&ev_file->event_list,
2675                                         struct devx_event_subscription,
2676                                         event_list);
2677                 eventsz = sizeof(event_sub->cookie);
2678                 event_data = &event_sub->cookie;
2679         } else {
2680                 event = list_first_entry(&ev_file->event_list,
2681                                       struct devx_async_event_data, list);
2682                 eventsz = sizeof(struct mlx5_eqe) +
2683                         sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2684                 event_data = &event->hdr;
2685         }
2686
2687         if (eventsz > count) {
2688                 spin_unlock_irq(&ev_file->lock);
2689                 return -EINVAL;
2690         }
2691
2692         if (omit_data)
2693                 list_del_init(&event_sub->event_list);
2694         else
2695                 list_del(&event->list);
2696
2697         spin_unlock_irq(&ev_file->lock);
2698
2699         if (copy_to_user(buf, event_data, eventsz))
2700                 /* This points to an application issue, not a kernel concern */
2701                 ret = -EFAULT;
2702         else
2703                 ret = eventsz;
2704
2705         if (!omit_data)
2706                 kfree(event);
2707         return ret;
2708 }
2709
2710 static __poll_t devx_async_event_poll(struct file *filp,
2711                                       struct poll_table_struct *wait)
2712 {
2713         struct devx_async_event_file *ev_file = filp->private_data;
2714         __poll_t pollflags = 0;
2715
2716         poll_wait(filp, &ev_file->poll_wait, wait);
2717
2718         spin_lock_irq(&ev_file->lock);
2719         if (ev_file->is_destroyed)
2720                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2721         else if (!list_empty(&ev_file->event_list))
2722                 pollflags = EPOLLIN | EPOLLRDNORM;
2723         spin_unlock_irq(&ev_file->lock);
2724
2725         return pollflags;
2726 }
2727
2728 static void devx_free_subscription(struct rcu_head *rcu)
2729 {
2730         struct devx_event_subscription *event_sub =
2731                 container_of(rcu, struct devx_event_subscription, rcu);
2732
2733         if (event_sub->eventfd)
2734                 eventfd_ctx_put(event_sub->eventfd);
2735         uverbs_uobject_put(&event_sub->ev_file->uobj);
2736         kfree(event_sub);
2737 }
2738
2739 static const struct file_operations devx_async_event_fops = {
2740         .owner   = THIS_MODULE,
2741         .read    = devx_async_event_read,
2742         .poll    = devx_async_event_poll,
2743         .release = uverbs_uobject_fd_release,
2744         .llseek  = no_llseek,
2745 };
2746
2747 static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2748                                               enum rdma_remove_reason why)
2749 {
2750         struct devx_async_cmd_event_file *comp_ev_file =
2751                 container_of(uobj, struct devx_async_cmd_event_file,
2752                              uobj);
2753         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2754         struct devx_async_data *entry, *tmp;
2755
2756         spin_lock_irq(&ev_queue->lock);
2757         ev_queue->is_destroyed = 1;
2758         spin_unlock_irq(&ev_queue->lock);
2759         wake_up_interruptible(&ev_queue->poll_wait);
2760
2761         mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2762
2763         spin_lock_irq(&comp_ev_file->ev_queue.lock);
2764         list_for_each_entry_safe(entry, tmp,
2765                                  &comp_ev_file->ev_queue.event_list, list) {
2766                 list_del(&entry->list);
2767                 kvfree(entry);
2768         }
2769         spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2770 };
2771
2772 static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2773                                           enum rdma_remove_reason why)
2774 {
2775         struct devx_async_event_file *ev_file =
2776                 container_of(uobj, struct devx_async_event_file,
2777                              uobj);
2778         struct devx_event_subscription *event_sub, *event_sub_tmp;
2779         struct mlx5_ib_dev *dev = ev_file->dev;
2780
2781         spin_lock_irq(&ev_file->lock);
2782         ev_file->is_destroyed = 1;
2783
2784         /* free the pending events allocation */
2785         if (ev_file->omit_data) {
2786                 struct devx_event_subscription *event_sub, *tmp;
2787
2788                 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2789                                          event_list)
2790                         list_del_init(&event_sub->event_list);
2791
2792         } else {
2793                 struct devx_async_event_data *entry, *tmp;
2794
2795                 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2796                                          list) {
2797                         list_del(&entry->list);
2798                         kfree(entry);
2799                 }
2800         }
2801
2802         spin_unlock_irq(&ev_file->lock);
2803         wake_up_interruptible(&ev_file->poll_wait);
2804
2805         mutex_lock(&dev->devx_event_table.event_xa_lock);
2806         /* delete the subscriptions which are related to this FD */
2807         list_for_each_entry_safe(event_sub, event_sub_tmp,
2808                                  &ev_file->subscribed_events_list, file_list) {
2809                 devx_cleanup_subscription(dev, event_sub);
2810                 list_del_rcu(&event_sub->file_list);
2811                 /* subscription may not be used by the read API any more */
2812                 call_rcu(&event_sub->rcu, devx_free_subscription);
2813         }
2814         mutex_unlock(&dev->devx_event_table.event_xa_lock);
2815
2816         put_device(&dev->ib_dev.dev);
2817 };
2818
2819 DECLARE_UVERBS_NAMED_METHOD(
2820         MLX5_IB_METHOD_DEVX_UMEM_REG,
2821         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2822                         MLX5_IB_OBJECT_DEVX_UMEM,
2823                         UVERBS_ACCESS_NEW,
2824                         UA_MANDATORY),
2825         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2826                            UVERBS_ATTR_TYPE(u64),
2827                            UA_MANDATORY),
2828         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2829                            UVERBS_ATTR_TYPE(u64),
2830                            UA_MANDATORY),
2831         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2832                              enum ib_access_flags),
2833         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2834                              u64),
2835         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2836                             UVERBS_ATTR_TYPE(u32),
2837                             UA_MANDATORY));
2838
2839 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2840         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2841         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2842                         MLX5_IB_OBJECT_DEVX_UMEM,
2843                         UVERBS_ACCESS_DESTROY,
2844                         UA_MANDATORY));
2845
2846 DECLARE_UVERBS_NAMED_METHOD(
2847         MLX5_IB_METHOD_DEVX_QUERY_EQN,
2848         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2849                            UVERBS_ATTR_TYPE(u32),
2850                            UA_MANDATORY),
2851         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2852                             UVERBS_ATTR_TYPE(u32),
2853                             UA_MANDATORY));
2854
2855 DECLARE_UVERBS_NAMED_METHOD(
2856         MLX5_IB_METHOD_DEVX_QUERY_UAR,
2857         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2858                            UVERBS_ATTR_TYPE(u32),
2859                            UA_MANDATORY),
2860         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2861                             UVERBS_ATTR_TYPE(u32),
2862                             UA_MANDATORY));
2863
2864 DECLARE_UVERBS_NAMED_METHOD(
2865         MLX5_IB_METHOD_DEVX_OTHER,
2866         UVERBS_ATTR_PTR_IN(
2867                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2868                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2869                 UA_MANDATORY,
2870                 UA_ALLOC_AND_COPY),
2871         UVERBS_ATTR_PTR_OUT(
2872                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2873                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2874                 UA_MANDATORY));
2875
2876 DECLARE_UVERBS_NAMED_METHOD(
2877         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2878         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2879                         MLX5_IB_OBJECT_DEVX_OBJ,
2880                         UVERBS_ACCESS_NEW,
2881                         UA_MANDATORY),
2882         UVERBS_ATTR_PTR_IN(
2883                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2884                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2885                 UA_MANDATORY,
2886                 UA_ALLOC_AND_COPY),
2887         UVERBS_ATTR_PTR_OUT(
2888                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2889                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2890                 UA_MANDATORY));
2891
2892 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2893         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2894         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2895                         MLX5_IB_OBJECT_DEVX_OBJ,
2896                         UVERBS_ACCESS_DESTROY,
2897                         UA_MANDATORY));
2898
2899 DECLARE_UVERBS_NAMED_METHOD(
2900         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2901         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
2902                         UVERBS_IDR_ANY_OBJECT,
2903                         UVERBS_ACCESS_WRITE,
2904                         UA_MANDATORY),
2905         UVERBS_ATTR_PTR_IN(
2906                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2907                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2908                 UA_MANDATORY,
2909                 UA_ALLOC_AND_COPY),
2910         UVERBS_ATTR_PTR_OUT(
2911                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2912                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2913                 UA_MANDATORY));
2914
2915 DECLARE_UVERBS_NAMED_METHOD(
2916         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2917         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2918                         UVERBS_IDR_ANY_OBJECT,
2919                         UVERBS_ACCESS_READ,
2920                         UA_MANDATORY),
2921         UVERBS_ATTR_PTR_IN(
2922                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2923                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2924                 UA_MANDATORY,
2925                 UA_ALLOC_AND_COPY),
2926         UVERBS_ATTR_PTR_OUT(
2927                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2928                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2929                 UA_MANDATORY));
2930
2931 DECLARE_UVERBS_NAMED_METHOD(
2932         MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2933         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2934                         UVERBS_IDR_ANY_OBJECT,
2935                         UVERBS_ACCESS_READ,
2936                         UA_MANDATORY),
2937         UVERBS_ATTR_PTR_IN(
2938                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2939                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2940                 UA_MANDATORY,
2941                 UA_ALLOC_AND_COPY),
2942         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2943                 u16, UA_MANDATORY),
2944         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2945                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2946                 UVERBS_ACCESS_READ,
2947                 UA_MANDATORY),
2948         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2949                 UVERBS_ATTR_TYPE(u64),
2950                 UA_MANDATORY));
2951
2952 DECLARE_UVERBS_NAMED_METHOD(
2953         MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2954         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2955                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2956                 UVERBS_ACCESS_READ,
2957                 UA_MANDATORY),
2958         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2959                 MLX5_IB_OBJECT_DEVX_OBJ,
2960                 UVERBS_ACCESS_READ,
2961                 UA_OPTIONAL),
2962         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2963                 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2964                 UA_MANDATORY,
2965                 UA_ALLOC_AND_COPY),
2966         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2967                 UVERBS_ATTR_TYPE(u64),
2968                 UA_OPTIONAL),
2969         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2970                 UVERBS_ATTR_TYPE(u32),
2971                 UA_OPTIONAL));
2972
2973 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
2974                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2975                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
2976                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2977                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
2978
2979 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
2980                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2981                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2982                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2983                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
2984                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2985                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
2986
2987 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
2988                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2989                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2990                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
2991
2992
2993 DECLARE_UVERBS_NAMED_METHOD(
2994         MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2995         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2996                         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2997                         UVERBS_ACCESS_NEW,
2998                         UA_MANDATORY));
2999
3000 DECLARE_UVERBS_NAMED_OBJECT(
3001         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3002         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
3003                              devx_async_cmd_event_destroy_uobj,
3004                              &devx_async_cmd_event_fops, "[devx_async_cmd]",
3005                              O_RDONLY),
3006         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
3007
3008 DECLARE_UVERBS_NAMED_METHOD(
3009         MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
3010         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
3011                         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3012                         UVERBS_ACCESS_NEW,
3013                         UA_MANDATORY),
3014         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
3015                         enum mlx5_ib_uapi_devx_create_event_channel_flags,
3016                         UA_MANDATORY));
3017
3018 DECLARE_UVERBS_NAMED_OBJECT(
3019         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3020         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
3021                              devx_async_event_destroy_uobj,
3022                              &devx_async_event_fops, "[devx_async_event]",
3023                              O_RDONLY),
3024         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
3025
3026 static bool devx_is_supported(struct ib_device *device)
3027 {
3028         struct mlx5_ib_dev *dev = to_mdev(device);
3029
3030         return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
3031 }
3032
3033 const struct uapi_definition mlx5_ib_devx_defs[] = {
3034         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3035                 MLX5_IB_OBJECT_DEVX,
3036                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3037         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3038                 MLX5_IB_OBJECT_DEVX_OBJ,
3039                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3040         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3041                 MLX5_IB_OBJECT_DEVX_UMEM,
3042                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3043         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3044                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3045                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3046         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3047                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3048                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3049         {},
3050 };