drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/mlx5_user_ioctl_verbs.h>
  12 #include <rdma/ib_umem.h>
  13 #include <rdma/uverbs_std_types.h>
  14 #include <linux/mlx5/driver.h>
  15 #include <linux/mlx5/fs.h>
  16 #include "mlx5_ib.h"
  17 #include <linux/xarray.h>
  18
  19 #define UVERBS_MODULE_NAME mlx5_ib
  20 #include <rdma/uverbs_named_ioctl.h>
  21
  22 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
  23
  24 enum devx_obj_flags {
  25         DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  26         DEVX_OBJ_FLAGS_DCT = 1 << 1,
  27         DEVX_OBJ_FLAGS_CQ = 1 << 2,
  28 };
  29
  30 struct devx_async_data {
  31         struct mlx5_ib_dev *mdev;
  32         struct list_head list;
  33         struct ib_uobject *fd_uobj;
  34         struct mlx5_async_work cb_work;
  35         u16 cmd_out_len;
  36         /* must be last field in this structure */
  37         struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  38 };
  39
  40 struct devx_async_event_data {
  41         struct list_head list; /* headed in ev_file->event_list */
  42         struct mlx5_ib_uapi_devx_async_event_hdr hdr;
  43 };
  44
  45 /* first level XA value data structure */
  46 struct devx_event {
  47         struct xarray object_ids; /* second XA level, Key = object id */
  48         struct list_head unaffiliated_list;
  49 };
  50
  51 /* second level XA value data structure */
  52 struct devx_obj_event {
  53         struct rcu_head rcu;
  54         struct list_head obj_sub_list;
  55 };
  56
  57 struct devx_event_subscription {
  58         struct list_head file_list; /* headed in ev_file->
  59                                      * subscribed_events_list
  60                                      */
  61         struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
  62                                    * devx_obj_event->obj_sub_list
  63                                    */
  64         struct list_head obj_list; /* headed in devx_object */
  65         struct list_head event_list; /* headed in ev_file->event_list or in
  66                                       * temp list via subscription
  67                                       */
  68
  69         u8 is_cleaned:1;
  70         u32 xa_key_level1;
  71         u32 xa_key_level2;
  72         struct rcu_head rcu;
  73         u64 cookie;
  74         struct devx_async_event_file *ev_file;
  75         struct file *filp; /* Upon hot unplug we need a direct access to */
  76         struct eventfd_ctx *eventfd;
  77 };
  78
  79 struct devx_async_event_file {
  80         struct ib_uobject uobj;
  81         /* Head of events that are subscribed to this FD */
  82         struct list_head subscribed_events_list;
  83         spinlock_t lock;
  84         wait_queue_head_t poll_wait;
  85         struct list_head event_list;
  86         struct mlx5_ib_dev *dev;
  87         u8 omit_data:1;
  88         u8 is_overflow_err:1;
  89         u8 is_destroyed:1;
  90 };
  91
  92 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
  93 struct devx_obj {
  94         struct mlx5_ib_dev      *ib_dev;
  95         u64                     obj_id;
  96         u32                     dinlen; /* destroy inbox length */
  97         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
  98         u32                     flags;
  99         union {
 100                 struct mlx5_ib_devx_mr  devx_mr;
 101                 struct mlx5_core_dct    core_dct;
 102                 struct mlx5_core_cq     core_cq;
 103         };
 104         struct list_head event_sub; /* holds devx_event_subscription entries */
 105 };
 106
 107 struct devx_umem {
 108         struct mlx5_core_dev            *mdev;
 109         struct ib_umem                  *umem;
 110         u32                             page_offset;
 111         int                             page_shift;
 112         int                             ncont;
 113         u32                             dinlen;
 114         u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
 115 };
 116
 117 struct devx_umem_reg_cmd {
 118         void                            *in;
 119         u32                             inlen;
 120         u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 121 };
 122
 123 static struct mlx5_ib_ucontext *
 124 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
 125 {
 126         return to_mucontext(ib_uverbs_get_ucontext(attrs));
 127 }
 128
 129 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
 130 {
 131         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
 132         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 133         void *uctx;
 134         int err;
 135         u16 uid;
 136         u32 cap = 0;
 137
 138         /* 0 means not supported */
 139         if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
 140                 return -EINVAL;
 141
 142         uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
 143         if (is_user && capable(CAP_NET_RAW) &&
 144             (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
 145                 cap |= MLX5_UCTX_CAP_RAW_TX;
 146         if (is_user && capable(CAP_SYS_RAWIO) &&
 147             (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
 148              MLX5_UCTX_CAP_INTERNAL_DEV_RES))
 149                 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
 150
 151         MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
 152         MLX5_SET(uctx, uctx, cap, cap);
 153
 154         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 155         if (err)
 156                 return err;
 157
 158         uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 159         return uid;
 160 }
 161
 162 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 163 {
 164         u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
 165         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 166
 167         MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 168         MLX5_SET(destroy_uctx_in, in, uid, uid);
 169
 170         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 171 }
 172
 173 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
 174 {
 175         struct devx_obj *devx_obj = obj;
 176         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 177
 178         switch (opcode) {
 179         case MLX5_CMD_OP_DESTROY_TIR:
 180                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 181                 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
 182                                     obj_id);
 183                 return true;
 184
 185         case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 186                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 187                 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
 188                                     table_id);
 189                 return true;
 190         default:
 191                 return false;
 192         }
 193 }
 194
 195 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id)
 196 {
 197         struct devx_obj *devx_obj = obj;
 198         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
 199
 200         if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
 201                 *counter_id = MLX5_GET(dealloc_flow_counter_in,
 202                                        devx_obj->dinbox,
 203                                        flow_counter_id);
 204                 return true;
 205         }
 206
 207         return false;
 208 }
 209
 210 static bool is_legacy_unaffiliated_event_num(u16 event_num)
 211 {
 212         switch (event_num) {
 213         case MLX5_EVENT_TYPE_PORT_CHANGE:
 214                 return true;
 215         default:
 216                 return false;
 217         }
 218 }
 219
 220 static bool is_legacy_obj_event_num(u16 event_num)
 221 {
 222         switch (event_num) {
 223         case MLX5_EVENT_TYPE_PATH_MIG:
 224         case MLX5_EVENT_TYPE_COMM_EST:
 225         case MLX5_EVENT_TYPE_SQ_DRAINED:
 226         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 227         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 228         case MLX5_EVENT_TYPE_CQ_ERROR:
 229         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 230         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 231         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 232         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 233         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 234         case MLX5_EVENT_TYPE_DCT_DRAINED:
 235         case MLX5_EVENT_TYPE_COMP:
 236         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 237         case MLX5_EVENT_TYPE_XRQ_ERROR:
 238                 return true;
 239         default:
 240                 return false;
 241         }
 242 }
 243
 244 static u16 get_legacy_obj_type(u16 opcode)
 245 {
 246         switch (opcode) {
 247         case MLX5_CMD_OP_CREATE_RQ:
 248                 return MLX5_EVENT_QUEUE_TYPE_RQ;
 249         case MLX5_CMD_OP_CREATE_QP:
 250                 return MLX5_EVENT_QUEUE_TYPE_QP;
 251         case MLX5_CMD_OP_CREATE_SQ:
 252                 return MLX5_EVENT_QUEUE_TYPE_SQ;
 253         case MLX5_CMD_OP_CREATE_DCT:
 254                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 255         default:
 256                 return 0;
 257         }
 258 }
 259
 260 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
 261 {
 262         u16 opcode;
 263
 264         opcode = (obj->obj_id >> 32) & 0xffff;
 265
 266         if (is_legacy_obj_event_num(event_num))
 267                 return get_legacy_obj_type(opcode);
 268
 269         switch (opcode) {
 270         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 271                 return (obj->obj_id >> 48);
 272         case MLX5_CMD_OP_CREATE_RQ:
 273                 return MLX5_OBJ_TYPE_RQ;
 274         case MLX5_CMD_OP_CREATE_QP:
 275                 return MLX5_OBJ_TYPE_QP;
 276         case MLX5_CMD_OP_CREATE_SQ:
 277                 return MLX5_OBJ_TYPE_SQ;
 278         case MLX5_CMD_OP_CREATE_DCT:
 279                 return MLX5_OBJ_TYPE_DCT;
 280         case MLX5_CMD_OP_CREATE_TIR:
 281                 return MLX5_OBJ_TYPE_TIR;
 282         case MLX5_CMD_OP_CREATE_TIS:
 283                 return MLX5_OBJ_TYPE_TIS;
 284         case MLX5_CMD_OP_CREATE_PSV:
 285                 return MLX5_OBJ_TYPE_PSV;
 286         case MLX5_OBJ_TYPE_MKEY:
 287                 return MLX5_OBJ_TYPE_MKEY;
 288         case MLX5_CMD_OP_CREATE_RMP:
 289                 return MLX5_OBJ_TYPE_RMP;
 290         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 291                 return MLX5_OBJ_TYPE_XRC_SRQ;
 292         case MLX5_CMD_OP_CREATE_XRQ:
 293                 return MLX5_OBJ_TYPE_XRQ;
 294         case MLX5_CMD_OP_CREATE_RQT:
 295                 return MLX5_OBJ_TYPE_RQT;
 296         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 297                 return MLX5_OBJ_TYPE_FLOW_COUNTER;
 298         case MLX5_CMD_OP_CREATE_CQ:
 299                 return MLX5_OBJ_TYPE_CQ;
 300         default:
 301                 return 0;
 302         }
 303 }
 304
 305 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
 306 {
 307         switch (event_type) {
 308         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 309         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 310         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 311         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 312         case MLX5_EVENT_TYPE_PATH_MIG:
 313         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 314         case MLX5_EVENT_TYPE_COMM_EST:
 315         case MLX5_EVENT_TYPE_SQ_DRAINED:
 316         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 317         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 318                 return eqe->data.qp_srq.type;
 319         case MLX5_EVENT_TYPE_CQ_ERROR:
 320         case MLX5_EVENT_TYPE_XRQ_ERROR:
 321                 return 0;
 322         case MLX5_EVENT_TYPE_DCT_DRAINED:
 323         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 324                 return MLX5_EVENT_QUEUE_TYPE_DCT;
 325         default:
 326                 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
 327         }
 328 }
 329
 330 static u32 get_dec_obj_id(u64 obj_id)
 331 {
 332         return (obj_id & 0xffffffff);
 333 }
 334
 335 /*
 336  * As the obj_id in the firmware is not globally unique the object type
 337  * must be considered upon checking for a valid object id.
 338  * For that the opcode of the creator command is encoded as part of the obj_id.
 339  */
 340 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
 341 {
 342         return ((u64)opcode << 32) | obj_id;
 343 }
 344
 345 static u64 devx_get_obj_id(const void *in)
 346 {
 347         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 348         u64 obj_id;
 349
 350         switch (opcode) {
 351         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 352         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 353                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
 354                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 355                                                  obj_type) << 16,
 356                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 357                                                  obj_id));
 358                 break;
 359         case MLX5_CMD_OP_QUERY_MKEY:
 360                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 361                                         MLX5_GET(query_mkey_in, in,
 362                                                  mkey_index));
 363                 break;
 364         case MLX5_CMD_OP_QUERY_CQ:
 365                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 366                                         MLX5_GET(query_cq_in, in, cqn));
 367                 break;
 368         case MLX5_CMD_OP_MODIFY_CQ:
 369                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 370                                         MLX5_GET(modify_cq_in, in, cqn));
 371                 break;
 372         case MLX5_CMD_OP_QUERY_SQ:
 373                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 374                                         MLX5_GET(query_sq_in, in, sqn));
 375                 break;
 376         case MLX5_CMD_OP_MODIFY_SQ:
 377                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 378                                         MLX5_GET(modify_sq_in, in, sqn));
 379                 break;
 380         case MLX5_CMD_OP_QUERY_RQ:
 381                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 382                                         MLX5_GET(query_rq_in, in, rqn));
 383                 break;
 384         case MLX5_CMD_OP_MODIFY_RQ:
 385                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 386                                         MLX5_GET(modify_rq_in, in, rqn));
 387                 break;
 388         case MLX5_CMD_OP_QUERY_RMP:
 389                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 390                                         MLX5_GET(query_rmp_in, in, rmpn));
 391                 break;
 392         case MLX5_CMD_OP_MODIFY_RMP:
 393                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 394                                         MLX5_GET(modify_rmp_in, in, rmpn));
 395                 break;
 396         case MLX5_CMD_OP_QUERY_RQT:
 397                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 398                                         MLX5_GET(query_rqt_in, in, rqtn));
 399                 break;
 400         case MLX5_CMD_OP_MODIFY_RQT:
 401                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 402                                         MLX5_GET(modify_rqt_in, in, rqtn));
 403                 break;
 404         case MLX5_CMD_OP_QUERY_TIR:
 405                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 406                                         MLX5_GET(query_tir_in, in, tirn));
 407                 break;
 408         case MLX5_CMD_OP_MODIFY_TIR:
 409                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 410                                         MLX5_GET(modify_tir_in, in, tirn));
 411                 break;
 412         case MLX5_CMD_OP_QUERY_TIS:
 413                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 414                                         MLX5_GET(query_tis_in, in, tisn));
 415                 break;
 416         case MLX5_CMD_OP_MODIFY_TIS:
 417                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 418                                         MLX5_GET(modify_tis_in, in, tisn));
 419                 break;
 420         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 421                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 422                                         MLX5_GET(query_flow_table_in, in,
 423                                                  table_id));
 424                 break;
 425         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 426                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 427                                         MLX5_GET(modify_flow_table_in, in,
 428                                                  table_id));
 429                 break;
 430         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 431                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 432                                         MLX5_GET(query_flow_group_in, in,
 433                                                  group_id));
 434                 break;
 435         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 436                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 437                                         MLX5_GET(query_fte_in, in,
 438                                                  flow_index));
 439                 break;
 440         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 441                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 442                                         MLX5_GET(set_fte_in, in, flow_index));
 443                 break;
 444         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 445                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 446                                         MLX5_GET(query_q_counter_in, in,
 447                                                  counter_set_id));
 448                 break;
 449         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 450                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 451                                         MLX5_GET(query_flow_counter_in, in,
 452                                                  flow_counter_id));
 453                 break;
 454         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 455                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 456                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 457                                                  obj_id));
 458                 break;
 459         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 460                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 461                                         MLX5_GET(query_scheduling_element_in,
 462                                                  in, scheduling_element_id));
 463                 break;
 464         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 465                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 466                                         MLX5_GET(modify_scheduling_element_in,
 467                                                  in, scheduling_element_id));
 468                 break;
 469         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 470                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 471                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 472                                                  vxlan_udp_port));
 473                 break;
 474         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 475                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 476                                         MLX5_GET(query_l2_table_entry_in, in,
 477                                                  table_index));
 478                 break;
 479         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 480                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 481                                         MLX5_GET(set_l2_table_entry_in, in,
 482                                                  table_index));
 483                 break;
 484         case MLX5_CMD_OP_QUERY_QP:
 485                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 486                                         MLX5_GET(query_qp_in, in, qpn));
 487                 break;
 488         case MLX5_CMD_OP_RST2INIT_QP:
 489                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 490                                         MLX5_GET(rst2init_qp_in, in, qpn));
 491                 break;
 492         case MLX5_CMD_OP_INIT2RTR_QP:
 493                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 494                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 495                 break;
 496         case MLX5_CMD_OP_RTR2RTS_QP:
 497                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 498                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 499                 break;
 500         case MLX5_CMD_OP_RTS2RTS_QP:
 501                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 502                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 503                 break;
 504         case MLX5_CMD_OP_SQERR2RTS_QP:
 505                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 506                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 507                 break;
 508         case MLX5_CMD_OP_2ERR_QP:
 509                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 510                                         MLX5_GET(qp_2err_in, in, qpn));
 511                 break;
 512         case MLX5_CMD_OP_2RST_QP:
 513                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 514                                         MLX5_GET(qp_2rst_in, in, qpn));
 515                 break;
 516         case MLX5_CMD_OP_QUERY_DCT:
 517                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 518                                         MLX5_GET(query_dct_in, in, dctn));
 519                 break;
 520         case MLX5_CMD_OP_QUERY_XRQ:
 521         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 522         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 523                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 524                                         MLX5_GET(query_xrq_in, in, xrqn));
 525                 break;
 526         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 527                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 528                                         MLX5_GET(query_xrc_srq_in, in,
 529                                                  xrc_srqn));
 530                 break;
 531         case MLX5_CMD_OP_ARM_XRC_SRQ:
 532                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 533                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 534                 break;
 535         case MLX5_CMD_OP_QUERY_SRQ:
 536                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 537                                         MLX5_GET(query_srq_in, in, srqn));
 538                 break;
 539         case MLX5_CMD_OP_ARM_RQ:
 540                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 541                                         MLX5_GET(arm_rq_in, in, srq_number));
 542                 break;
 543         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 544                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 545                                         MLX5_GET(drain_dct_in, in, dctn));
 546                 break;
 547         case MLX5_CMD_OP_ARM_XRQ:
 548         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 549         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 550         case MLX5_CMD_OP_MODIFY_XRQ:
 551                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 552                                         MLX5_GET(arm_xrq_in, in, xrqn));
 553                 break;
 554         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 555                 obj_id = get_enc_obj_id
 556                                 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 557                                  MLX5_GET(query_packet_reformat_context_in,
 558                                           in, packet_reformat_id));
 559                 break;
 560         default:
 561                 obj_id = 0;
 562         }
 563
 564         return obj_id;
 565 }
 566
 567 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 568                                  struct ib_uobject *uobj, const void *in)
 569 {
 570         struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 571         u64 obj_id = devx_get_obj_id(in);
 572
 573         if (!obj_id)
 574                 return false;
 575
 576         switch (uobj_get_object_id(uobj)) {
 577         case UVERBS_OBJECT_CQ:
 578                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 579                                       to_mcq(uobj->object)->mcq.cqn) ==
 580                                       obj_id;
 581
 582         case UVERBS_OBJECT_SRQ:
 583         {
 584                 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 585                 u16 opcode;
 586
 587                 switch (srq->common.res) {
 588                 case MLX5_RES_XSRQ:
 589                         opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 590                         break;
 591                 case MLX5_RES_XRQ:
 592                         opcode = MLX5_CMD_OP_CREATE_XRQ;
 593                         break;
 594                 default:
 595                         if (!dev->mdev->issi)
 596                                 opcode = MLX5_CMD_OP_CREATE_SRQ;
 597                         else
 598                                 opcode = MLX5_CMD_OP_CREATE_RMP;
 599                 }
 600
 601                 return get_enc_obj_id(opcode,
 602                                       to_msrq(uobj->object)->msrq.srqn) ==
 603                                       obj_id;
 604         }
 605
 606         case UVERBS_OBJECT_QP:
 607         {
 608                 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 609                 enum ib_qp_type qp_type = qp->ibqp.qp_type;
 610
 611                 if (qp_type == IB_QPT_RAW_PACKET ||
 612                     (qp->flags & MLX5_IB_QP_UNDERLAY)) {
 613                         struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 614                                                          &qp->raw_packet_qp;
 615                         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 616                         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 617
 618                         return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 619                                                rq->base.mqp.qpn) == obj_id ||
 620                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 621                                                sq->base.mqp.qpn) == obj_id ||
 622                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 623                                                rq->tirn) == obj_id ||
 624                                 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 625                                                sq->tisn) == obj_id);
 626                 }
 627
 628                 if (qp_type == MLX5_IB_QPT_DCT)
 629                         return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 630                                               qp->dct.mdct.mqp.qpn) == obj_id;
 631
 632                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 633                                       qp->ibqp.qp_num) == obj_id;
 634         }
 635
 636         case UVERBS_OBJECT_WQ:
 637                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 638                                       to_mrwq(uobj->object)->core_qp.qpn) ==
 639                                       obj_id;
 640
 641         case UVERBS_OBJECT_RWQ_IND_TBL:
 642                 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 643                                       to_mrwq_ind_table(uobj->object)->rqtn) ==
 644                                       obj_id;
 645
 646         case MLX5_IB_OBJECT_DEVX_OBJ:
 647                 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
 648
 649         default:
 650                 return false;
 651         }
 652 }
 653
 654 static void devx_set_umem_valid(const void *in)
 655 {
 656         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 657
 658         switch (opcode) {
 659         case MLX5_CMD_OP_CREATE_MKEY:
 660                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 661                 break;
 662         case MLX5_CMD_OP_CREATE_CQ:
 663         {
 664                 void *cqc;
 665
 666                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 667                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 668                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 669                 break;
 670         }
 671         case MLX5_CMD_OP_CREATE_QP:
 672         {
 673                 void *qpc;
 674
 675                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 676                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 677                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 678                 break;
 679         }
 680
 681         case MLX5_CMD_OP_CREATE_RQ:
 682         {
 683                 void *rqc, *wq;
 684
 685                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 686                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 687                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 688                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 689                 break;
 690         }
 691
 692         case MLX5_CMD_OP_CREATE_SQ:
 693         {
 694                 void *sqc, *wq;
 695
 696                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 697                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 698                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 699                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 700                 break;
 701         }
 702
 703         case MLX5_CMD_OP_MODIFY_CQ:
 704                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 705                 break;
 706
 707         case MLX5_CMD_OP_CREATE_RMP:
 708         {
 709                 void *rmpc, *wq;
 710
 711                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 712                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 713                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 714                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 715                 break;
 716         }
 717
 718         case MLX5_CMD_OP_CREATE_XRQ:
 719         {
 720                 void *xrqc, *wq;
 721
 722                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 723                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 724                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 725                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 726                 break;
 727         }
 728
 729         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 730         {
 731                 void *xrc_srqc;
 732
 733                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 734                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 735                                         xrc_srq_context_entry);
 736                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 737                 break;
 738         }
 739
 740         default:
 741                 return;
 742         }
 743 }
 744
 745 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 746 {
 747         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 748
 749         switch (*opcode) {
 750         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 751         case MLX5_CMD_OP_CREATE_MKEY:
 752         case MLX5_CMD_OP_CREATE_CQ:
 753         case MLX5_CMD_OP_ALLOC_PD:
 754         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 755         case MLX5_CMD_OP_CREATE_RMP:
 756         case MLX5_CMD_OP_CREATE_SQ:
 757         case MLX5_CMD_OP_CREATE_RQ:
 758         case MLX5_CMD_OP_CREATE_RQT:
 759         case MLX5_CMD_OP_CREATE_TIR:
 760         case MLX5_CMD_OP_CREATE_TIS:
 761         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 762         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 763         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 764         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 765         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 766         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 767         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 768         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 769         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 770         case MLX5_CMD_OP_CREATE_QP:
 771         case MLX5_CMD_OP_CREATE_SRQ:
 772         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 773         case MLX5_CMD_OP_CREATE_DCT:
 774         case MLX5_CMD_OP_CREATE_XRQ:
 775         case MLX5_CMD_OP_ATTACH_TO_MCG:
 776         case MLX5_CMD_OP_ALLOC_XRCD:
 777                 return true;
 778         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 779         {
 780                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 781                 if (op_mod == 0)
 782                         return true;
 783                 return false;
 784         }
 785         case MLX5_CMD_OP_CREATE_PSV:
 786         {
 787                 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
 788
 789                 if (num_psv == 1)
 790                         return true;
 791                 return false;
 792         }
 793         default:
 794                 return false;
 795         }
 796 }
 797
 798 static bool devx_is_obj_modify_cmd(const void *in)
 799 {
 800         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 801
 802         switch (opcode) {
 803         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 804         case MLX5_CMD_OP_MODIFY_CQ:
 805         case MLX5_CMD_OP_MODIFY_RMP:
 806         case MLX5_CMD_OP_MODIFY_SQ:
 807         case MLX5_CMD_OP_MODIFY_RQ:
 808         case MLX5_CMD_OP_MODIFY_RQT:
 809         case MLX5_CMD_OP_MODIFY_TIR:
 810         case MLX5_CMD_OP_MODIFY_TIS:
 811         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 812         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 813         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 814         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 815         case MLX5_CMD_OP_RST2INIT_QP:
 816         case MLX5_CMD_OP_INIT2RTR_QP:
 817         case MLX5_CMD_OP_RTR2RTS_QP:
 818         case MLX5_CMD_OP_RTS2RTS_QP:
 819         case MLX5_CMD_OP_SQERR2RTS_QP:
 820         case MLX5_CMD_OP_2ERR_QP:
 821         case MLX5_CMD_OP_2RST_QP:
 822         case MLX5_CMD_OP_ARM_XRC_SRQ:
 823         case MLX5_CMD_OP_ARM_RQ:
 824         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 825         case MLX5_CMD_OP_ARM_XRQ:
 826         case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 827         case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 828         case MLX5_CMD_OP_MODIFY_XRQ:
 829                 return true;
 830         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 831         {
 832                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 833
 834                 if (op_mod == 1)
 835                         return true;
 836                 return false;
 837         }
 838         default:
 839                 return false;
 840         }
 841 }
 842
 843 static bool devx_is_obj_query_cmd(const void *in)
 844 {
 845         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 846
 847         switch (opcode) {
 848         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 849         case MLX5_CMD_OP_QUERY_MKEY:
 850         case MLX5_CMD_OP_QUERY_CQ:
 851         case MLX5_CMD_OP_QUERY_RMP:
 852         case MLX5_CMD_OP_QUERY_SQ:
 853         case MLX5_CMD_OP_QUERY_RQ:
 854         case MLX5_CMD_OP_QUERY_RQT:
 855         case MLX5_CMD_OP_QUERY_TIR:
 856         case MLX5_CMD_OP_QUERY_TIS:
 857         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 858         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 859         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 860         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 861         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 862         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 863         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 864         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 865         case MLX5_CMD_OP_QUERY_QP:
 866         case MLX5_CMD_OP_QUERY_SRQ:
 867         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 868         case MLX5_CMD_OP_QUERY_DCT:
 869         case MLX5_CMD_OP_QUERY_XRQ:
 870         case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 871         case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 872         case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 873                 return true;
 874         default:
 875                 return false;
 876         }
 877 }
 878
 879 static bool devx_is_whitelist_cmd(void *in)
 880 {
 881         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 882
 883         switch (opcode) {
 884         case MLX5_CMD_OP_QUERY_HCA_CAP:
 885         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 886         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 887                 return true;
 888         default:
 889                 return false;
 890         }
 891 }
 892
 893 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 894 {
 895         if (devx_is_whitelist_cmd(cmd_in)) {
 896                 struct mlx5_ib_dev *dev;
 897
 898                 if (c->devx_uid)
 899                         return c->devx_uid;
 900
 901                 dev = to_mdev(c->ibucontext.device);
 902                 if (dev->devx_whitelist_uid)
 903                         return dev->devx_whitelist_uid;
 904
 905                 return -EOPNOTSUPP;
 906         }
 907
 908         if (!c->devx_uid)
 909                 return -EINVAL;
 910
 911         return c->devx_uid;
 912 }
 913
 914 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
 915 {
 916         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 917
 918         /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
 919         if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
 920              MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
 921             (opcode >= MLX5_CMD_OP_GENERAL_START &&
 922              opcode < MLX5_CMD_OP_GENERAL_END))
 923                 return true;
 924
 925         switch (opcode) {
 926         case MLX5_CMD_OP_QUERY_HCA_CAP:
 927         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 928         case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 929         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 930         case MLX5_CMD_OP_QUERY_ADAPTER:
 931         case MLX5_CMD_OP_QUERY_ISSI:
 932         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 933         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 934         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 935         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 936         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 937         case MLX5_CMD_OP_NOP:
 938         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 939         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 940         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 941         case MLX5_CMD_OP_QUERY_LAG:
 942                 return true;
 943         default:
 944                 return false;
 945         }
 946 }
 947
 948 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 949         struct uverbs_attr_bundle *attrs)
 950 {
 951         struct mlx5_ib_ucontext *c;
 952         struct mlx5_ib_dev *dev;
 953         int user_vector;
 954         int dev_eqn;
 955         unsigned int irqn;
 956         int err;
 957
 958         if (uverbs_copy_from(&user_vector, attrs,
 959                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 960                 return -EFAULT;
 961
 962         c = devx_ufile2uctx(attrs);
 963         if (IS_ERR(c))
 964                 return PTR_ERR(c);
 965         dev = to_mdev(c->ibucontext.device);
 966
 967         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
 968         if (err < 0)
 969                 return err;
 970
 971         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 972                            &dev_eqn, sizeof(dev_eqn)))
 973                 return -EFAULT;
 974
 975         return 0;
 976 }
 977
 978 /*
 979  *Security note:
 980  * The hardware protection mechanism works like this: Each device object that
 981  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
 982  * the device specification manual) upon its creation. Then upon doorbell,
 983  * hardware fetches the object context for which the doorbell was rang, and
 984  * validates that the UAR through which the DB was rang matches the UAR ID
 985  * of the object.
 986  * If no match the doorbell is silently ignored by the hardware. Of course,
 987  * the user cannot ring a doorbell on a UAR that was not mapped to it.
 988  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
 989  * mailboxes (except tagging them with UID), we expose to the user its UAR
 990  * ID, so it can embed it in these objects in the expected specification
 991  * format. So the only thing the user can do is hurt itself by creating a
 992  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
 993  * may ring a doorbell on its objects.
 994  * The consequence of that will be that another user can schedule a QP/SQ
 995  * of the buggy user for execution (just insert it to the hardware schedule
 996  * queue or arm its CQ for event generation), no further harm is expected.
 997  */
 998 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
 999         struct uverbs_attr_bundle *attrs)
1000 {
1001         struct mlx5_ib_ucontext *c;
1002         struct mlx5_ib_dev *dev;
1003         u32 user_idx;
1004         s32 dev_idx;
1005
1006         c = devx_ufile2uctx(attrs);
1007         if (IS_ERR(c))
1008                 return PTR_ERR(c);
1009         dev = to_mdev(c->ibucontext.device);
1010
1011         if (uverbs_copy_from(&user_idx, attrs,
1012                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1013                 return -EFAULT;
1014
1015         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1016         if (dev_idx < 0)
1017                 return dev_idx;
1018
1019         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1020                            &dev_idx, sizeof(dev_idx)))
1021                 return -EFAULT;
1022
1023         return 0;
1024 }
1025
1026 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1027         struct uverbs_attr_bundle *attrs)
1028 {
1029         struct mlx5_ib_ucontext *c;
1030         struct mlx5_ib_dev *dev;
1031         void *cmd_in = uverbs_attr_get_alloced_ptr(
1032                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1033         int cmd_out_len = uverbs_attr_get_len(attrs,
1034                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1035         void *cmd_out;
1036         int err;
1037         int uid;
1038
1039         c = devx_ufile2uctx(attrs);
1040         if (IS_ERR(c))
1041                 return PTR_ERR(c);
1042         dev = to_mdev(c->ibucontext.device);
1043
1044         uid = devx_get_uid(c, cmd_in);
1045         if (uid < 0)
1046                 return uid;
1047
1048         /* Only white list of some general HCA commands are allowed for this method. */
1049         if (!devx_is_general_cmd(cmd_in, dev))
1050                 return -EINVAL;
1051
1052         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1053         if (IS_ERR(cmd_out))
1054                 return PTR_ERR(cmd_out);
1055
1056         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1057         err = mlx5_cmd_exec(dev->mdev, cmd_in,
1058                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1059                             cmd_out, cmd_out_len);
1060         if (err)
1061                 return err;
1062
1063         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1064                               cmd_out_len);
1065 }
1066
1067 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1068                                        u32 *dinlen,
1069                                        u32 *obj_id)
1070 {
1071         u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
1072         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1073
1074         *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1075         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1076
1077         MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1078         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1079
1080         switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
1081         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1082                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1083                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
1084                 break;
1085
1086         case MLX5_CMD_OP_CREATE_UMEM:
1087                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1088                          MLX5_CMD_OP_DESTROY_UMEM);
1089                 break;
1090         case MLX5_CMD_OP_CREATE_MKEY:
1091                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
1092                 break;
1093         case MLX5_CMD_OP_CREATE_CQ:
1094                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1095                 break;
1096         case MLX5_CMD_OP_ALLOC_PD:
1097                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1098                 break;
1099         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1100                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1101                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1102                 break;
1103         case MLX5_CMD_OP_CREATE_RMP:
1104                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1105                 break;
1106         case MLX5_CMD_OP_CREATE_SQ:
1107                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1108                 break;
1109         case MLX5_CMD_OP_CREATE_RQ:
1110                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1111                 break;
1112         case MLX5_CMD_OP_CREATE_RQT:
1113                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1114                 break;
1115         case MLX5_CMD_OP_CREATE_TIR:
1116                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1117                 break;
1118         case MLX5_CMD_OP_CREATE_TIS:
1119                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1120                 break;
1121         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1122                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1123                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1124                 break;
1125         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1126                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1127                 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
1128                 MLX5_SET(destroy_flow_table_in, din, other_vport,
1129                          MLX5_GET(create_flow_table_in,  in, other_vport));
1130                 MLX5_SET(destroy_flow_table_in, din, vport_number,
1131                          MLX5_GET(create_flow_table_in,  in, vport_number));
1132                 MLX5_SET(destroy_flow_table_in, din, table_type,
1133                          MLX5_GET(create_flow_table_in,  in, table_type));
1134                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1135                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1136                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1137                 break;
1138         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1139                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1140                 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
1141                 MLX5_SET(destroy_flow_group_in, din, other_vport,
1142                          MLX5_GET(create_flow_group_in, in, other_vport));
1143                 MLX5_SET(destroy_flow_group_in, din, vport_number,
1144                          MLX5_GET(create_flow_group_in, in, vport_number));
1145                 MLX5_SET(destroy_flow_group_in, din, table_type,
1146                          MLX5_GET(create_flow_group_in, in, table_type));
1147                 MLX5_SET(destroy_flow_group_in, din, table_id,
1148                          MLX5_GET(create_flow_group_in, in, table_id));
1149                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1150                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1151                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1152                 break;
1153         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1154                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1155                 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
1156                 MLX5_SET(delete_fte_in, din, other_vport,
1157                          MLX5_GET(set_fte_in,  in, other_vport));
1158                 MLX5_SET(delete_fte_in, din, vport_number,
1159                          MLX5_GET(set_fte_in, in, vport_number));
1160                 MLX5_SET(delete_fte_in, din, table_type,
1161                          MLX5_GET(set_fte_in, in, table_type));
1162                 MLX5_SET(delete_fte_in, din, table_id,
1163                          MLX5_GET(set_fte_in, in, table_id));
1164                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1165                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1166                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1167                 break;
1168         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1169                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1170                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1171                 break;
1172         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1173                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1174                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1175                 break;
1176         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1177                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1178                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1179                 break;
1180         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1181                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1182                 *obj_id = MLX5_GET(create_scheduling_element_out, out,
1183                                    scheduling_element_id);
1184                 MLX5_SET(destroy_scheduling_element_in, din,
1185                          scheduling_hierarchy,
1186                          MLX5_GET(create_scheduling_element_in, in,
1187                                   scheduling_hierarchy));
1188                 MLX5_SET(destroy_scheduling_element_in, din,
1189                          scheduling_element_id, *obj_id);
1190                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1191                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1192                 break;
1193         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1194                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1195                 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
1196                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1197                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1198                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1199                 break;
1200         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1201                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1202                 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1203                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1204                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1205                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1206                 break;
1207         case MLX5_CMD_OP_CREATE_QP:
1208                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1209                 break;
1210         case MLX5_CMD_OP_CREATE_SRQ:
1211                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1212                 break;
1213         case MLX5_CMD_OP_CREATE_XRC_SRQ:
1214                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1215                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
1216                 break;
1217         case MLX5_CMD_OP_CREATE_DCT:
1218                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1219                 break;
1220         case MLX5_CMD_OP_CREATE_XRQ:
1221                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1222                 break;
1223         case MLX5_CMD_OP_ATTACH_TO_MCG:
1224                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1225                 MLX5_SET(detach_from_mcg_in, din, qpn,
1226                          MLX5_GET(attach_to_mcg_in, in, qpn));
1227                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1228                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1229                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1230                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1231                 break;
1232         case MLX5_CMD_OP_ALLOC_XRCD:
1233                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1234                 break;
1235         case MLX5_CMD_OP_CREATE_PSV:
1236                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1237                          MLX5_CMD_OP_DESTROY_PSV);
1238                 MLX5_SET(destroy_psv_in, din, psvn,
1239                          MLX5_GET(create_psv_out, out, psv0_index));
1240                 break;
1241         default:
1242                 /* The entry must match to one of the devx_is_obj_create_cmd */
1243                 WARN_ON(true);
1244                 break;
1245         }
1246 }
1247
1248 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1249                                      struct mlx5_ib_dev *dev,
1250                                      void *in, void *out)
1251 {
1252         struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1253         struct mlx5_core_mkey *mkey;
1254         void *mkc;
1255         u8 key;
1256
1257         mkey = &devx_mr->mmkey;
1258         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1259         key = MLX5_GET(mkc, mkc, mkey_7_0);
1260         mkey->key = mlx5_idx_to_mkey(
1261                         MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1262         mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1263         mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1264         mkey->size = MLX5_GET64(mkc, mkc, len);
1265         mkey->pd = MLX5_GET(mkc, mkc, pd);
1266         devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1267
1268         return xa_err(xa_store(&dev->mdev->priv.mkey_table,
1269                                mlx5_base_mkey(mkey->key), mkey, GFP_KERNEL));
1270 }
1271
1272 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1273                                    struct devx_obj *obj,
1274                                    void *in, int in_len)
1275 {
1276         int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1277                         MLX5_FLD_SZ_BYTES(create_mkey_in,
1278                         memory_key_mkey_entry);
1279         void *mkc;
1280         u8 access_mode;
1281
1282         if (in_len < min_len)
1283                 return -EINVAL;
1284
1285         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1286
1287         access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1288         access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1289
1290         if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1291                 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1292                 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1293                         obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1294                 return 0;
1295         }
1296
1297         MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1298         return 0;
1299 }
1300
1301 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1302                                       struct devx_event_subscription *sub)
1303 {
1304         struct devx_event *event;
1305         struct devx_obj_event *xa_val_level2;
1306
1307         if (sub->is_cleaned)
1308                 return;
1309
1310         sub->is_cleaned = 1;
1311         list_del_rcu(&sub->xa_list);
1312
1313         if (list_empty(&sub->obj_list))
1314                 return;
1315
1316         list_del_rcu(&sub->obj_list);
1317         /* check whether key level 1 for this obj_sub_list is empty */
1318         event = xa_load(&dev->devx_event_table.event_xa,
1319                         sub->xa_key_level1);
1320         WARN_ON(!event);
1321
1322         xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1323         if (list_empty(&xa_val_level2->obj_sub_list)) {
1324                 xa_erase(&event->object_ids,
1325                          sub->xa_key_level2);
1326                 kfree_rcu(xa_val_level2, rcu);
1327         }
1328 }
1329
1330 static int devx_obj_cleanup(struct ib_uobject *uobject,
1331                             enum rdma_remove_reason why,
1332                             struct uverbs_attr_bundle *attrs)
1333 {
1334         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1335         struct mlx5_devx_event_table *devx_event_table;
1336         struct devx_obj *obj = uobject->object;
1337         struct devx_event_subscription *sub_entry, *tmp;
1338         struct mlx5_ib_dev *dev;
1339         int ret;
1340
1341         dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1342         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1343                 /*
1344                  * The pagefault_single_data_segment() does commands against
1345                  * the mmkey, we must wait for that to stop before freeing the
1346                  * mkey, as another allocation could get the same mkey #.
1347                  */
1348                 xa_erase(&obj->ib_dev->mdev->priv.mkey_table,
1349                          mlx5_base_mkey(obj->devx_mr.mmkey.key));
1350                 synchronize_srcu(&dev->mr_srcu);
1351         }
1352
1353         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1354                 ret = mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
1355         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1356                 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1357         else
1358                 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1359                                     obj->dinlen, out, sizeof(out));
1360         if (ib_is_destroy_retryable(ret, why, uobject))
1361                 return ret;
1362
1363         devx_event_table = &dev->devx_event_table;
1364
1365         mutex_lock(&devx_event_table->event_xa_lock);
1366         list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1367                 devx_cleanup_subscription(dev, sub_entry);
1368         mutex_unlock(&devx_event_table->event_xa_lock);
1369
1370         kfree(obj);
1371         return ret;
1372 }
1373
1374 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1375 {
1376         struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1377         struct mlx5_devx_event_table *table;
1378         struct devx_event *event;
1379         struct devx_obj_event *obj_event;
1380         u32 obj_id = mcq->cqn;
1381
1382         table = &obj->ib_dev->devx_event_table;
1383         rcu_read_lock();
1384         event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1385         if (!event)
1386                 goto out;
1387
1388         obj_event = xa_load(&event->object_ids, obj_id);
1389         if (!obj_event)
1390                 goto out;
1391
1392         dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1393 out:
1394         rcu_read_unlock();
1395 }
1396
1397 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1398         struct uverbs_attr_bundle *attrs)
1399 {
1400         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1401         int cmd_out_len =  uverbs_attr_get_len(attrs,
1402                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1403         int cmd_in_len = uverbs_attr_get_len(attrs,
1404                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1405         void *cmd_out;
1406         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1407                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1408         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1409                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1410         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1411         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1412         struct devx_obj *obj;
1413         u16 obj_type = 0;
1414         int err;
1415         int uid;
1416         u32 obj_id;
1417         u16 opcode;
1418
1419         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1420                 return -EINVAL;
1421
1422         uid = devx_get_uid(c, cmd_in);
1423         if (uid < 0)
1424                 return uid;
1425
1426         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1427                 return -EINVAL;
1428
1429         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1430         if (IS_ERR(cmd_out))
1431                 return PTR_ERR(cmd_out);
1432
1433         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1434         if (!obj)
1435                 return -ENOMEM;
1436
1437         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1438         if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1439                 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1440                 if (err)
1441                         goto obj_free;
1442         } else {
1443                 devx_set_umem_valid(cmd_in);
1444         }
1445
1446         if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1447                 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1448                 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct,
1449                                            cmd_in, cmd_in_len,
1450                                            cmd_out, cmd_out_len);
1451         } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
1452                 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1453                 obj->core_cq.comp = devx_cq_comp;
1454                 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1455                                           cmd_in, cmd_in_len, cmd_out,
1456                                           cmd_out_len);
1457         } else {
1458                 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1459                                     cmd_in_len,
1460                                     cmd_out, cmd_out_len);
1461         }
1462
1463         if (err)
1464                 goto obj_free;
1465
1466         uobj->object = obj;
1467         INIT_LIST_HEAD(&obj->event_sub);
1468         obj->ib_dev = dev;
1469         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1470                                    &obj_id);
1471         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1472
1473         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1474         if (err)
1475                 goto obj_destroy;
1476
1477         if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1478                 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1479         obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1480
1481         if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1482                 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1483                 if (err)
1484                         goto obj_destroy;
1485         }
1486         return 0;
1487
1488 obj_destroy:
1489         if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1490                 mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
1491         else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1492                 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1493         else
1494                 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1495                               sizeof(out));
1496 obj_free:
1497         kfree(obj);
1498         return err;
1499 }
1500
1501 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1502         struct uverbs_attr_bundle *attrs)
1503 {
1504         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1505         int cmd_out_len = uverbs_attr_get_len(attrs,
1506                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1507         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1508                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1509         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1510                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1511         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1512         void *cmd_out;
1513         int err;
1514         int uid;
1515
1516         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1517                 return -EINVAL;
1518
1519         uid = devx_get_uid(c, cmd_in);
1520         if (uid < 0)
1521                 return uid;
1522
1523         if (!devx_is_obj_modify_cmd(cmd_in))
1524                 return -EINVAL;
1525
1526         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1527                 return -EINVAL;
1528
1529         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1530         if (IS_ERR(cmd_out))
1531                 return PTR_ERR(cmd_out);
1532
1533         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1534         devx_set_umem_valid(cmd_in);
1535
1536         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1537                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1538                             cmd_out, cmd_out_len);
1539         if (err)
1540                 return err;
1541
1542         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1543                               cmd_out, cmd_out_len);
1544 }
1545
1546 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1547         struct uverbs_attr_bundle *attrs)
1548 {
1549         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1550         int cmd_out_len = uverbs_attr_get_len(attrs,
1551                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1552         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1553                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1554         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1555                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1556         void *cmd_out;
1557         int err;
1558         int uid;
1559         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1560
1561         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1562                 return -EINVAL;
1563
1564         uid = devx_get_uid(c, cmd_in);
1565         if (uid < 0)
1566                 return uid;
1567
1568         if (!devx_is_obj_query_cmd(cmd_in))
1569                 return -EINVAL;
1570
1571         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1572                 return -EINVAL;
1573
1574         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1575         if (IS_ERR(cmd_out))
1576                 return PTR_ERR(cmd_out);
1577
1578         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1579         err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1580                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1581                             cmd_out, cmd_out_len);
1582         if (err)
1583                 return err;
1584
1585         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1586                               cmd_out, cmd_out_len);
1587 }
1588
1589 struct devx_async_event_queue {
1590         spinlock_t              lock;
1591         wait_queue_head_t       poll_wait;
1592         struct list_head        event_list;
1593         atomic_t                bytes_in_use;
1594         u8                      is_destroyed:1;
1595 };
1596
1597 struct devx_async_cmd_event_file {
1598         struct ib_uobject               uobj;
1599         struct devx_async_event_queue   ev_queue;
1600         struct mlx5_async_ctx           async_ctx;
1601 };
1602
1603 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1604 {
1605         spin_lock_init(&ev_queue->lock);
1606         INIT_LIST_HEAD(&ev_queue->event_list);
1607         init_waitqueue_head(&ev_queue->poll_wait);
1608         atomic_set(&ev_queue->bytes_in_use, 0);
1609         ev_queue->is_destroyed = 0;
1610 }
1611
1612 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1613         struct uverbs_attr_bundle *attrs)
1614 {
1615         struct devx_async_cmd_event_file *ev_file;
1616
1617         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1618                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1619         struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1620
1621         ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1622                                uobj);
1623         devx_init_event_queue(&ev_file->ev_queue);
1624         mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1625         return 0;
1626 }
1627
1628 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1629         struct uverbs_attr_bundle *attrs)
1630 {
1631         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1632                 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1633         struct devx_async_event_file *ev_file;
1634         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1635                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1636         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1637         u32 flags;
1638         int err;
1639
1640         err = uverbs_get_flags32(&flags, attrs,
1641                 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1642                 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1643
1644         if (err)
1645                 return err;
1646
1647         ev_file = container_of(uobj, struct devx_async_event_file,
1648                                uobj);
1649         spin_lock_init(&ev_file->lock);
1650         INIT_LIST_HEAD(&ev_file->event_list);
1651         init_waitqueue_head(&ev_file->poll_wait);
1652         if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1653                 ev_file->omit_data = 1;
1654         INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1655         ev_file->dev = dev;
1656         get_device(&dev->ib_dev.dev);
1657         return 0;
1658 }
1659
1660 static void devx_query_callback(int status, struct mlx5_async_work *context)
1661 {
1662         struct devx_async_data *async_data =
1663                 container_of(context, struct devx_async_data, cb_work);
1664         struct ib_uobject *fd_uobj = async_data->fd_uobj;
1665         struct devx_async_cmd_event_file *ev_file;
1666         struct devx_async_event_queue *ev_queue;
1667         unsigned long flags;
1668
1669         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1670                                uobj);
1671         ev_queue = &ev_file->ev_queue;
1672
1673         spin_lock_irqsave(&ev_queue->lock, flags);
1674         list_add_tail(&async_data->list, &ev_queue->event_list);
1675         spin_unlock_irqrestore(&ev_queue->lock, flags);
1676
1677         wake_up_interruptible(&ev_queue->poll_wait);
1678         fput(fd_uobj->object);
1679 }
1680
1681 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1682
1683 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1684         struct uverbs_attr_bundle *attrs)
1685 {
1686         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1687                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1688         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1689                                 attrs,
1690                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1691         u16 cmd_out_len;
1692         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1693                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1694         struct ib_uobject *fd_uobj;
1695         int err;
1696         int uid;
1697         struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1698         struct devx_async_cmd_event_file *ev_file;
1699         struct devx_async_data *async_data;
1700
1701         if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1702                 return -EINVAL;
1703
1704         uid = devx_get_uid(c, cmd_in);
1705         if (uid < 0)
1706                 return uid;
1707
1708         if (!devx_is_obj_query_cmd(cmd_in))
1709                 return -EINVAL;
1710
1711         err = uverbs_get_const(&cmd_out_len, attrs,
1712                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1713         if (err)
1714                 return err;
1715
1716         if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1717                 return -EINVAL;
1718
1719         fd_uobj = uverbs_attr_get_uobject(attrs,
1720                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1721         if (IS_ERR(fd_uobj))
1722                 return PTR_ERR(fd_uobj);
1723
1724         ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1725                                uobj);
1726
1727         if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1728                         MAX_ASYNC_BYTES_IN_USE) {
1729                 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1730                 return -EAGAIN;
1731         }
1732
1733         async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1734                                           cmd_out_len), GFP_KERNEL);
1735         if (!async_data) {
1736                 err = -ENOMEM;
1737                 goto sub_bytes;
1738         }
1739
1740         err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1741                                MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1742         if (err)
1743                 goto free_async;
1744
1745         async_data->cmd_out_len = cmd_out_len;
1746         async_data->mdev = mdev;
1747         async_data->fd_uobj = fd_uobj;
1748
1749         get_file(fd_uobj->object);
1750         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1751         err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1752                     uverbs_attr_get_len(attrs,
1753                                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1754                     async_data->hdr.out_data,
1755                     async_data->cmd_out_len,
1756                     devx_query_callback, &async_data->cb_work);
1757
1758         if (err)
1759                 goto cb_err;
1760
1761         return 0;
1762
1763 cb_err:
1764         fput(fd_uobj->object);
1765 free_async:
1766         kvfree(async_data);
1767 sub_bytes:
1768         atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1769         return err;
1770 }
1771
1772 static void
1773 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1774                            u32 key_level1,
1775                            bool is_level2,
1776                            u32 key_level2)
1777 {
1778         struct devx_event *event;
1779         struct devx_obj_event *xa_val_level2;
1780
1781         /* Level 1 is valid for future use, no need to free */
1782         if (!is_level2)
1783                 return;
1784
1785         event = xa_load(&devx_event_table->event_xa, key_level1);
1786         WARN_ON(!event);
1787
1788         xa_val_level2 = xa_load(&event->object_ids,
1789                                 key_level2);
1790         if (list_empty(&xa_val_level2->obj_sub_list)) {
1791                 xa_erase(&event->object_ids,
1792                          key_level2);
1793                 kfree_rcu(xa_val_level2, rcu);
1794         }
1795 }
1796
1797 static int
1798 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1799                          u32 key_level1,
1800                          bool is_level2,
1801                          u32 key_level2)
1802 {
1803         struct devx_obj_event *obj_event;
1804         struct devx_event *event;
1805         int err;
1806
1807         event = xa_load(&devx_event_table->event_xa, key_level1);
1808         if (!event) {
1809                 event = kzalloc(sizeof(*event), GFP_KERNEL);
1810                 if (!event)
1811                         return -ENOMEM;
1812
1813                 INIT_LIST_HEAD(&event->unaffiliated_list);
1814                 xa_init(&event->object_ids);
1815
1816                 err = xa_insert(&devx_event_table->event_xa,
1817                                 key_level1,
1818                                 event,
1819                                 GFP_KERNEL);
1820                 if (err) {
1821                         kfree(event);
1822                         return err;
1823                 }
1824         }
1825
1826         if (!is_level2)
1827                 return 0;
1828
1829         obj_event = xa_load(&event->object_ids, key_level2);
1830         if (!obj_event) {
1831                 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1832                 if (!obj_event)
1833                         /* Level1 is valid for future use, no need to free */
1834                         return -ENOMEM;
1835
1836                 err = xa_insert(&event->object_ids,
1837                                 key_level2,
1838                                 obj_event,
1839                                 GFP_KERNEL);
1840                 if (err)
1841                         return err;
1842                 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1843         }
1844
1845         return 0;
1846 }
1847
1848 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1849                                    struct devx_obj *obj)
1850 {
1851         int i;
1852
1853         for (i = 0; i < num_events; i++) {
1854                 if (obj) {
1855                         if (!is_legacy_obj_event_num(event_type_num_list[i]))
1856                                 return false;
1857                 } else if (!is_legacy_unaffiliated_event_num(
1858                                 event_type_num_list[i])) {
1859                         return false;
1860                 }
1861         }
1862
1863         return true;
1864 }
1865
1866 #define MAX_SUPP_EVENT_NUM 255
1867 static bool is_valid_events(struct mlx5_core_dev *dev,
1868                             int num_events, u16 *event_type_num_list,
1869                             struct devx_obj *obj)
1870 {
1871         __be64 *aff_events;
1872         __be64 *unaff_events;
1873         int mask_entry;
1874         int mask_bit;
1875         int i;
1876
1877         if (MLX5_CAP_GEN(dev, event_cap)) {
1878                 aff_events = MLX5_CAP_DEV_EVENT(dev,
1879                                                 user_affiliated_events);
1880                 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1881                                                   user_unaffiliated_events);
1882         } else {
1883                 return is_valid_events_legacy(num_events, event_type_num_list,
1884                                               obj);
1885         }
1886
1887         for (i = 0; i < num_events; i++) {
1888                 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1889                         return false;
1890
1891                 mask_entry = event_type_num_list[i] / 64;
1892                 mask_bit = event_type_num_list[i] % 64;
1893
1894                 if (obj) {
1895                         /* CQ completion */
1896                         if (event_type_num_list[i] == 0)
1897                                 continue;
1898
1899                         if (!(be64_to_cpu(aff_events[mask_entry]) &
1900                                         (1ull << mask_bit)))
1901                                 return false;
1902
1903                         continue;
1904                 }
1905
1906                 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1907                                 (1ull << mask_bit)))
1908                         return false;
1909         }
1910
1911         return true;
1912 }
1913
1914 #define MAX_NUM_EVENTS 16
1915 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1916         struct uverbs_attr_bundle *attrs)
1917 {
1918         struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1919                                 attrs,
1920                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1921         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1922                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1923         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1924         struct ib_uobject *fd_uobj;
1925         struct devx_obj *obj = NULL;
1926         struct devx_async_event_file *ev_file;
1927         struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1928         u16 *event_type_num_list;
1929         struct devx_event_subscription *event_sub, *tmp_sub;
1930         struct list_head sub_list;
1931         int redirect_fd;
1932         bool use_eventfd = false;
1933         int num_events;
1934         int num_alloc_xa_entries = 0;
1935         u16 obj_type = 0;
1936         u64 cookie = 0;
1937         u32 obj_id = 0;
1938         int err;
1939         int i;
1940
1941         if (!c->devx_uid)
1942                 return -EINVAL;
1943
1944         if (!IS_ERR(devx_uobj)) {
1945                 obj = (struct devx_obj *)devx_uobj->object;
1946                 if (obj)
1947                         obj_id = get_dec_obj_id(obj->obj_id);
1948         }
1949
1950         fd_uobj = uverbs_attr_get_uobject(attrs,
1951                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
1952         if (IS_ERR(fd_uobj))
1953                 return PTR_ERR(fd_uobj);
1954
1955         ev_file = container_of(fd_uobj, struct devx_async_event_file,
1956                                uobj);
1957
1958         if (uverbs_attr_is_valid(attrs,
1959                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
1960                 err = uverbs_copy_from(&redirect_fd, attrs,
1961                                MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
1962                 if (err)
1963                         return err;
1964
1965                 use_eventfd = true;
1966         }
1967
1968         if (uverbs_attr_is_valid(attrs,
1969                                  MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
1970                 if (use_eventfd)
1971                         return -EINVAL;
1972
1973                 err = uverbs_copy_from(&cookie, attrs,
1974                                 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
1975                 if (err)
1976                         return err;
1977         }
1978
1979         num_events = uverbs_attr_ptr_get_array_size(
1980                 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
1981                 sizeof(u16));
1982
1983         if (num_events < 0)
1984                 return num_events;
1985
1986         if (num_events > MAX_NUM_EVENTS)
1987                 return -EINVAL;
1988
1989         event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
1990                         MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
1991
1992         if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
1993                 return -EINVAL;
1994
1995         INIT_LIST_HEAD(&sub_list);
1996
1997         /* Protect from concurrent subscriptions to same XA entries to allow
1998          * both to succeed
1999          */
2000         mutex_lock(&devx_event_table->event_xa_lock);
2001         for (i = 0; i < num_events; i++) {
2002                 u32 key_level1;
2003
2004                 if (obj)
2005                         obj_type = get_dec_obj_type(obj,
2006                                                     event_type_num_list[i]);
2007                 key_level1 = event_type_num_list[i] | obj_type << 16;
2008
2009                 err = subscribe_event_xa_alloc(devx_event_table,
2010                                                key_level1,
2011                                                obj,
2012                                                obj_id);
2013                 if (err)
2014                         goto err;
2015
2016                 num_alloc_xa_entries++;
2017                 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2018                 if (!event_sub)
2019                         goto err;
2020
2021                 list_add_tail(&event_sub->event_list, &sub_list);
2022                 if (use_eventfd) {
2023                         event_sub->eventfd =
2024                                 eventfd_ctx_fdget(redirect_fd);
2025
2026                         if (IS_ERR(event_sub->eventfd)) {
2027                                 err = PTR_ERR(event_sub->eventfd);
2028                                 event_sub->eventfd = NULL;
2029                                 goto err;
2030                         }
2031                 }
2032
2033                 event_sub->cookie = cookie;
2034                 event_sub->ev_file = ev_file;
2035                 event_sub->filp = fd_uobj->object;
2036                 /* May be needed upon cleanup the devx object/subscription */
2037                 event_sub->xa_key_level1 = key_level1;
2038                 event_sub->xa_key_level2 = obj_id;
2039                 INIT_LIST_HEAD(&event_sub->obj_list);
2040         }
2041
2042         /* Once all the allocations and the XA data insertions were done we
2043          * can go ahead and add all the subscriptions to the relevant lists
2044          * without concern of a failure.
2045          */
2046         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2047                 struct devx_event *event;
2048                 struct devx_obj_event *obj_event;
2049
2050                 list_del_init(&event_sub->event_list);
2051
2052                 spin_lock_irq(&ev_file->lock);
2053                 list_add_tail_rcu(&event_sub->file_list,
2054                                   &ev_file->subscribed_events_list);
2055                 spin_unlock_irq(&ev_file->lock);
2056
2057                 event = xa_load(&devx_event_table->event_xa,
2058                                 event_sub->xa_key_level1);
2059                 WARN_ON(!event);
2060
2061                 if (!obj) {
2062                         list_add_tail_rcu(&event_sub->xa_list,
2063                                           &event->unaffiliated_list);
2064                         continue;
2065                 }
2066
2067                 obj_event = xa_load(&event->object_ids, obj_id);
2068                 WARN_ON(!obj_event);
2069                 list_add_tail_rcu(&event_sub->xa_list,
2070                                   &obj_event->obj_sub_list);
2071                 list_add_tail_rcu(&event_sub->obj_list,
2072                                   &obj->event_sub);
2073         }
2074
2075         mutex_unlock(&devx_event_table->event_xa_lock);
2076         return 0;
2077
2078 err:
2079         list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2080                 list_del(&event_sub->event_list);
2081
2082                 subscribe_event_xa_dealloc(devx_event_table,
2083                                            event_sub->xa_key_level1,
2084                                            obj,
2085                                            obj_id);
2086
2087                 if (event_sub->eventfd)
2088                         eventfd_ctx_put(event_sub->eventfd);
2089
2090                 kfree(event_sub);
2091         }
2092
2093         mutex_unlock(&devx_event_table->event_xa_lock);
2094         return err;
2095 }
2096
2097 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2098                          struct uverbs_attr_bundle *attrs,
2099                          struct devx_umem *obj)
2100 {
2101         u64 addr;
2102         size_t size;
2103         u32 access;
2104         int npages;
2105         int err;
2106         u32 page_mask;
2107
2108         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2109             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2110                 return -EFAULT;
2111
2112         err = uverbs_get_flags32(&access, attrs,
2113                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2114                                  IB_ACCESS_LOCAL_WRITE |
2115                                  IB_ACCESS_REMOTE_WRITE |
2116                                  IB_ACCESS_REMOTE_READ);
2117         if (err)
2118                 return err;
2119
2120         err = ib_check_mr_access(access);
2121         if (err)
2122                 return err;
2123
2124         obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0);
2125         if (IS_ERR(obj->umem))
2126                 return PTR_ERR(obj->umem);
2127
2128         mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2129                            MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2130                            &obj->page_shift, &obj->ncont, NULL);
2131
2132         if (!npages) {
2133                 ib_umem_release(obj->umem);
2134                 return -EINVAL;
2135         }
2136
2137         page_mask = (1 << obj->page_shift) - 1;
2138         obj->page_offset = obj->umem->address & page_mask;
2139
2140         return 0;
2141 }
2142
2143 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2144                                    struct devx_umem *obj,
2145                                    struct devx_umem_reg_cmd *cmd)
2146 {
2147         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2148                     (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
2149         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2150         return PTR_ERR_OR_ZERO(cmd->in);
2151 }
2152
2153 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2154                                     struct devx_umem *obj,
2155                                     struct devx_umem_reg_cmd *cmd)
2156 {
2157         void *umem;
2158         __be64 *mtt;
2159
2160         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2161         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2162
2163         MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2164         MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2165         MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2166                                             MLX5_ADAPTER_PAGE_SHIFT);
2167         MLX5_SET(umem, umem, page_offset, obj->page_offset);
2168         mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2169                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2170                              MLX5_IB_MTT_READ);
2171 }
2172
2173 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2174         struct uverbs_attr_bundle *attrs)
2175 {
2176         struct devx_umem_reg_cmd cmd;
2177         struct devx_umem *obj;
2178         struct ib_uobject *uobj = uverbs_attr_get_uobject(
2179                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2180         u32 obj_id;
2181         struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2182                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2183         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2184         int err;
2185
2186         if (!c->devx_uid)
2187                 return -EINVAL;
2188
2189         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2190         if (!obj)
2191                 return -ENOMEM;
2192
2193         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2194         if (err)
2195                 goto err_obj_free;
2196
2197         err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
2198         if (err)
2199                 goto err_umem_release;
2200
2201         devx_umem_reg_cmd_build(dev, obj, &cmd);
2202
2203         MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2204         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2205                             sizeof(cmd.out));
2206         if (err)
2207                 goto err_umem_release;
2208
2209         obj->mdev = dev->mdev;
2210         uobj->object = obj;
2211         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2212         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
2213         if (err)
2214                 goto err_umem_destroy;
2215
2216         return 0;
2217
2218 err_umem_destroy:
2219         mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
2220 err_umem_release:
2221         ib_umem_release(obj->umem);
2222 err_obj_free:
2223         kfree(obj);
2224         return err;
2225 }
2226
2227 static int devx_umem_cleanup(struct ib_uobject *uobject,
2228                              enum rdma_remove_reason why,
2229                              struct uverbs_attr_bundle *attrs)
2230 {
2231         struct devx_umem *obj = uobject->object;
2232         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2233         int err;
2234
2235         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2236         if (ib_is_destroy_retryable(err, why, uobject))
2237                 return err;
2238
2239         ib_umem_release(obj->umem);
2240         kfree(obj);
2241         return 0;
2242 }
2243
2244 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2245                                   unsigned long event_type)
2246 {
2247         __be64 *unaff_events;
2248         int mask_entry;
2249         int mask_bit;
2250
2251         if (!MLX5_CAP_GEN(dev, event_cap))
2252                 return is_legacy_unaffiliated_event_num(event_type);
2253
2254         unaff_events = MLX5_CAP_DEV_EVENT(dev,
2255                                           user_unaffiliated_events);
2256         WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2257
2258         mask_entry = event_type / 64;
2259         mask_bit = event_type % 64;
2260
2261         if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2262                 return false;
2263
2264         return true;
2265 }
2266
2267 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2268 {
2269         struct mlx5_eqe *eqe = data;
2270         u32 obj_id = 0;
2271
2272         switch (event_type) {
2273         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2274         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2275         case MLX5_EVENT_TYPE_PATH_MIG:
2276         case MLX5_EVENT_TYPE_COMM_EST:
2277         case MLX5_EVENT_TYPE_SQ_DRAINED:
2278         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2279         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2280         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2281         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2282         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2283                 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2284                 break;
2285         case MLX5_EVENT_TYPE_XRQ_ERROR:
2286                 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2287                 break;
2288         case MLX5_EVENT_TYPE_DCT_DRAINED:
2289         case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2290                 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2291                 break;
2292         case MLX5_EVENT_TYPE_CQ_ERROR:
2293                 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2294                 break;
2295         default:
2296                 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2297                 break;
2298         }
2299
2300         return obj_id;
2301 }
2302
2303 static int deliver_event(struct devx_event_subscription *event_sub,
2304                          const void *data)
2305 {
2306         struct devx_async_event_file *ev_file;
2307         struct devx_async_event_data *event_data;
2308         unsigned long flags;
2309
2310         ev_file = event_sub->ev_file;
2311
2312         if (ev_file->omit_data) {
2313                 spin_lock_irqsave(&ev_file->lock, flags);
2314                 if (!list_empty(&event_sub->event_list)) {
2315                         spin_unlock_irqrestore(&ev_file->lock, flags);
2316                         return 0;
2317                 }
2318
2319                 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2320                 spin_unlock_irqrestore(&ev_file->lock, flags);
2321                 wake_up_interruptible(&ev_file->poll_wait);
2322                 return 0;
2323         }
2324
2325         event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2326                              GFP_ATOMIC);
2327         if (!event_data) {
2328                 spin_lock_irqsave(&ev_file->lock, flags);
2329                 ev_file->is_overflow_err = 1;
2330                 spin_unlock_irqrestore(&ev_file->lock, flags);
2331                 return -ENOMEM;
2332         }
2333
2334         event_data->hdr.cookie = event_sub->cookie;
2335         memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2336
2337         spin_lock_irqsave(&ev_file->lock, flags);
2338         list_add_tail(&event_data->list, &ev_file->event_list);
2339         spin_unlock_irqrestore(&ev_file->lock, flags);
2340         wake_up_interruptible(&ev_file->poll_wait);
2341
2342         return 0;
2343 }
2344
2345 static void dispatch_event_fd(struct list_head *fd_list,
2346                               const void *data)
2347 {
2348         struct devx_event_subscription *item;
2349
2350         list_for_each_entry_rcu(item, fd_list, xa_list) {
2351                 if (!get_file_rcu(item->filp))
2352                         continue;
2353
2354                 if (item->eventfd) {
2355                         eventfd_signal(item->eventfd, 1);
2356                         fput(item->filp);
2357                         continue;
2358                 }
2359
2360                 deliver_event(item, data);
2361                 fput(item->filp);
2362         }
2363 }
2364
2365 static int devx_event_notifier(struct notifier_block *nb,
2366                                unsigned long event_type, void *data)
2367 {
2368         struct mlx5_devx_event_table *table;
2369         struct mlx5_ib_dev *dev;
2370         struct devx_event *event;
2371         struct devx_obj_event *obj_event;
2372         u16 obj_type = 0;
2373         bool is_unaffiliated;
2374         u32 obj_id;
2375
2376         /* Explicit filtering to kernel events which may occur frequently */
2377         if (event_type == MLX5_EVENT_TYPE_CMD ||
2378             event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2379                 return NOTIFY_OK;
2380
2381         table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2382         dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2383         is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2384
2385         if (!is_unaffiliated)
2386                 obj_type = get_event_obj_type(event_type, data);
2387
2388         rcu_read_lock();
2389         event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2390         if (!event) {
2391                 rcu_read_unlock();
2392                 return NOTIFY_DONE;
2393         }
2394
2395         if (is_unaffiliated) {
2396                 dispatch_event_fd(&event->unaffiliated_list, data);
2397                 rcu_read_unlock();
2398                 return NOTIFY_OK;
2399         }
2400
2401         obj_id = devx_get_obj_id_from_event(event_type, data);
2402         obj_event = xa_load(&event->object_ids, obj_id);
2403         if (!obj_event) {
2404                 rcu_read_unlock();
2405                 return NOTIFY_DONE;
2406         }
2407
2408         dispatch_event_fd(&obj_event->obj_sub_list, data);
2409
2410         rcu_read_unlock();
2411         return NOTIFY_OK;
2412 }
2413
2414 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
2415 {
2416         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2417
2418         xa_init(&table->event_xa);
2419         mutex_init(&table->event_xa_lock);
2420         MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2421         mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2422 }
2423
2424 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
2425 {
2426         struct mlx5_devx_event_table *table = &dev->devx_event_table;
2427         struct devx_event_subscription *sub, *tmp;
2428         struct devx_event *event;
2429         void *entry;
2430         unsigned long id;
2431
2432         mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2433         mutex_lock(&dev->devx_event_table.event_xa_lock);
2434         xa_for_each(&table->event_xa, id, entry) {
2435                 event = entry;
2436                 list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
2437                                          xa_list)
2438                         devx_cleanup_subscription(dev, sub);
2439                 kfree(entry);
2440         }
2441         mutex_unlock(&dev->devx_event_table.event_xa_lock);
2442         xa_destroy(&table->event_xa);
2443 }
2444
2445 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2446                                          size_t count, loff_t *pos)
2447 {
2448         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2449         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2450         struct devx_async_data *event;
2451         int ret = 0;
2452         size_t eventsz;
2453
2454         spin_lock_irq(&ev_queue->lock);
2455
2456         while (list_empty(&ev_queue->event_list)) {
2457                 spin_unlock_irq(&ev_queue->lock);
2458
2459                 if (filp->f_flags & O_NONBLOCK)
2460                         return -EAGAIN;
2461
2462                 if (wait_event_interruptible(
2463                             ev_queue->poll_wait,
2464                             (!list_empty(&ev_queue->event_list) ||
2465                              ev_queue->is_destroyed))) {
2466                         return -ERESTARTSYS;
2467                 }
2468
2469                 if (list_empty(&ev_queue->event_list) &&
2470                     ev_queue->is_destroyed)
2471                         return -EIO;
2472
2473                 spin_lock_irq(&ev_queue->lock);
2474         }
2475
2476         event = list_entry(ev_queue->event_list.next,
2477                            struct devx_async_data, list);
2478         eventsz = event->cmd_out_len +
2479                         sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2480
2481         if (eventsz > count) {
2482                 spin_unlock_irq(&ev_queue->lock);
2483                 return -ENOSPC;
2484         }
2485
2486         list_del(ev_queue->event_list.next);
2487         spin_unlock_irq(&ev_queue->lock);
2488
2489         if (copy_to_user(buf, &event->hdr, eventsz))
2490                 ret = -EFAULT;
2491         else
2492                 ret = eventsz;
2493
2494         atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2495         kvfree(event);
2496         return ret;
2497 }
2498
2499 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp)
2500 {
2501         struct ib_uobject *uobj = filp->private_data;
2502         struct devx_async_cmd_event_file *comp_ev_file = container_of(
2503                 uobj, struct devx_async_cmd_event_file, uobj);
2504         struct devx_async_data *entry, *tmp;
2505
2506         spin_lock_irq(&comp_ev_file->ev_queue.lock);
2507         list_for_each_entry_safe(entry, tmp,
2508                                  &comp_ev_file->ev_queue.event_list, list)
2509                 kvfree(entry);
2510         spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2511
2512         uverbs_close_fd(filp);
2513         return 0;
2514 }
2515
2516 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2517                                               struct poll_table_struct *wait)
2518 {
2519         struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2520         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2521         __poll_t pollflags = 0;
2522
2523         poll_wait(filp, &ev_queue->poll_wait, wait);
2524
2525         spin_lock_irq(&ev_queue->lock);
2526         if (ev_queue->is_destroyed)
2527                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2528         else if (!list_empty(&ev_queue->event_list))
2529                 pollflags = EPOLLIN | EPOLLRDNORM;
2530         spin_unlock_irq(&ev_queue->lock);
2531
2532         return pollflags;
2533 }
2534
2535 static const struct file_operations devx_async_cmd_event_fops = {
2536         .owner   = THIS_MODULE,
2537         .read    = devx_async_cmd_event_read,
2538         .poll    = devx_async_cmd_event_poll,
2539         .release = devx_async_cmd_event_close,
2540         .llseek  = no_llseek,
2541 };
2542
2543 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2544                                      size_t count, loff_t *pos)
2545 {
2546         struct devx_async_event_file *ev_file = filp->private_data;
2547         struct devx_event_subscription *event_sub;
2548         struct devx_async_event_data *uninitialized_var(event);
2549         int ret = 0;
2550         size_t eventsz;
2551         bool omit_data;
2552         void *event_data;
2553
2554         omit_data = ev_file->omit_data;
2555
2556         spin_lock_irq(&ev_file->lock);
2557
2558         if (ev_file->is_overflow_err) {
2559                 ev_file->is_overflow_err = 0;
2560                 spin_unlock_irq(&ev_file->lock);
2561                 return -EOVERFLOW;
2562         }
2563
2564         if (ev_file->is_destroyed) {
2565                 spin_unlock_irq(&ev_file->lock);
2566                 return -EIO;
2567         }
2568
2569         while (list_empty(&ev_file->event_list)) {
2570                 spin_unlock_irq(&ev_file->lock);
2571
2572                 if (filp->f_flags & O_NONBLOCK)
2573                         return -EAGAIN;
2574
2575                 if (wait_event_interruptible(ev_file->poll_wait,
2576                             (!list_empty(&ev_file->event_list) ||
2577                              ev_file->is_destroyed))) {
2578                         return -ERESTARTSYS;
2579                 }
2580
2581                 spin_lock_irq(&ev_file->lock);
2582                 if (ev_file->is_destroyed) {
2583                         spin_unlock_irq(&ev_file->lock);
2584                         return -EIO;
2585                 }
2586         }
2587
2588         if (omit_data) {
2589                 event_sub = list_first_entry(&ev_file->event_list,
2590                                         struct devx_event_subscription,
2591                                         event_list);
2592                 eventsz = sizeof(event_sub->cookie);
2593                 event_data = &event_sub->cookie;
2594         } else {
2595                 event = list_first_entry(&ev_file->event_list,
2596                                       struct devx_async_event_data, list);
2597                 eventsz = sizeof(struct mlx5_eqe) +
2598                         sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2599                 event_data = &event->hdr;
2600         }
2601
2602         if (eventsz > count) {
2603                 spin_unlock_irq(&ev_file->lock);
2604                 return -EINVAL;
2605         }
2606
2607         if (omit_data)
2608                 list_del_init(&event_sub->event_list);
2609         else
2610                 list_del(&event->list);
2611
2612         spin_unlock_irq(&ev_file->lock);
2613
2614         if (copy_to_user(buf, event_data, eventsz))
2615                 /* This points to an application issue, not a kernel concern */
2616                 ret = -EFAULT;
2617         else
2618                 ret = eventsz;
2619
2620         if (!omit_data)
2621                 kfree(event);
2622         return ret;
2623 }
2624
2625 static __poll_t devx_async_event_poll(struct file *filp,
2626                                       struct poll_table_struct *wait)
2627 {
2628         struct devx_async_event_file *ev_file = filp->private_data;
2629         __poll_t pollflags = 0;
2630
2631         poll_wait(filp, &ev_file->poll_wait, wait);
2632
2633         spin_lock_irq(&ev_file->lock);
2634         if (ev_file->is_destroyed)
2635                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2636         else if (!list_empty(&ev_file->event_list))
2637                 pollflags = EPOLLIN | EPOLLRDNORM;
2638         spin_unlock_irq(&ev_file->lock);
2639
2640         return pollflags;
2641 }
2642
2643 static int devx_async_event_close(struct inode *inode, struct file *filp)
2644 {
2645         struct devx_async_event_file *ev_file = filp->private_data;
2646         struct devx_event_subscription *event_sub, *event_sub_tmp;
2647         struct devx_async_event_data *entry, *tmp;
2648         struct mlx5_ib_dev *dev = ev_file->dev;
2649
2650         mutex_lock(&dev->devx_event_table.event_xa_lock);
2651         /* delete the subscriptions which are related to this FD */
2652         list_for_each_entry_safe(event_sub, event_sub_tmp,
2653                                  &ev_file->subscribed_events_list, file_list) {
2654                 devx_cleanup_subscription(dev, event_sub);
2655                 if (event_sub->eventfd)
2656                         eventfd_ctx_put(event_sub->eventfd);
2657
2658                 list_del_rcu(&event_sub->file_list);
2659                 /* subscription may not be used by the read API any more */
2660                 kfree_rcu(event_sub, rcu);
2661         }
2662
2663         mutex_unlock(&dev->devx_event_table.event_xa_lock);
2664
2665         /* free the pending events allocation */
2666         if (!ev_file->omit_data) {
2667                 spin_lock_irq(&ev_file->lock);
2668                 list_for_each_entry_safe(entry, tmp,
2669                                          &ev_file->event_list, list)
2670                         kfree(entry); /* read can't come any more */
2671                 spin_unlock_irq(&ev_file->lock);
2672         }
2673
2674         uverbs_close_fd(filp);
2675         put_device(&dev->ib_dev.dev);
2676         return 0;
2677 }
2678
2679 static const struct file_operations devx_async_event_fops = {
2680         .owner   = THIS_MODULE,
2681         .read    = devx_async_event_read,
2682         .poll    = devx_async_event_poll,
2683         .release = devx_async_event_close,
2684         .llseek  = no_llseek,
2685 };
2686
2687 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
2688                                                    enum rdma_remove_reason why)
2689 {
2690         struct devx_async_cmd_event_file *comp_ev_file =
2691                 container_of(uobj, struct devx_async_cmd_event_file,
2692                              uobj);
2693         struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2694
2695         spin_lock_irq(&ev_queue->lock);
2696         ev_queue->is_destroyed = 1;
2697         spin_unlock_irq(&ev_queue->lock);
2698
2699         if (why == RDMA_REMOVE_DRIVER_REMOVE)
2700                 wake_up_interruptible(&ev_queue->poll_wait);
2701
2702         mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2703         return 0;
2704 };
2705
2706 static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj,
2707                                             enum rdma_remove_reason why)
2708 {
2709         struct devx_async_event_file *ev_file =
2710                 container_of(uobj, struct devx_async_event_file,
2711                              uobj);
2712
2713         spin_lock_irq(&ev_file->lock);
2714         ev_file->is_destroyed = 1;
2715         spin_unlock_irq(&ev_file->lock);
2716
2717         wake_up_interruptible(&ev_file->poll_wait);
2718         return 0;
2719 };
2720
2721 DECLARE_UVERBS_NAMED_METHOD(
2722         MLX5_IB_METHOD_DEVX_UMEM_REG,
2723         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2724                         MLX5_IB_OBJECT_DEVX_UMEM,
2725                         UVERBS_ACCESS_NEW,
2726                         UA_MANDATORY),
2727         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2728                            UVERBS_ATTR_TYPE(u64),
2729                            UA_MANDATORY),
2730         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2731                            UVERBS_ATTR_TYPE(u64),
2732                            UA_MANDATORY),
2733         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2734                              enum ib_access_flags),
2735         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2736                             UVERBS_ATTR_TYPE(u32),
2737                             UA_MANDATORY));
2738
2739 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2740         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2741         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2742                         MLX5_IB_OBJECT_DEVX_UMEM,
2743                         UVERBS_ACCESS_DESTROY,
2744                         UA_MANDATORY));
2745
2746 DECLARE_UVERBS_NAMED_METHOD(
2747         MLX5_IB_METHOD_DEVX_QUERY_EQN,
2748         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2749                            UVERBS_ATTR_TYPE(u32),
2750                            UA_MANDATORY),
2751         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2752                             UVERBS_ATTR_TYPE(u32),
2753                             UA_MANDATORY));
2754
2755 DECLARE_UVERBS_NAMED_METHOD(
2756         MLX5_IB_METHOD_DEVX_QUERY_UAR,
2757         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2758                            UVERBS_ATTR_TYPE(u32),
2759                            UA_MANDATORY),
2760         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2761                             UVERBS_ATTR_TYPE(u32),
2762                             UA_MANDATORY));
2763
2764 DECLARE_UVERBS_NAMED_METHOD(
2765         MLX5_IB_METHOD_DEVX_OTHER,
2766         UVERBS_ATTR_PTR_IN(
2767                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2768                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2769                 UA_MANDATORY,
2770                 UA_ALLOC_AND_COPY),
2771         UVERBS_ATTR_PTR_OUT(
2772                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2773                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2774                 UA_MANDATORY));
2775
2776 DECLARE_UVERBS_NAMED_METHOD(
2777         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2778         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2779                         MLX5_IB_OBJECT_DEVX_OBJ,
2780                         UVERBS_ACCESS_NEW,
2781                         UA_MANDATORY),
2782         UVERBS_ATTR_PTR_IN(
2783                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2784                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2785                 UA_MANDATORY,
2786                 UA_ALLOC_AND_COPY),
2787         UVERBS_ATTR_PTR_OUT(
2788                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2789                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2790                 UA_MANDATORY));
2791
2792 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2793         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2794         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2795                         MLX5_IB_OBJECT_DEVX_OBJ,
2796                         UVERBS_ACCESS_DESTROY,
2797                         UA_MANDATORY));
2798
2799 DECLARE_UVERBS_NAMED_METHOD(
2800         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2801         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
2802                         UVERBS_IDR_ANY_OBJECT,
2803                         UVERBS_ACCESS_WRITE,
2804                         UA_MANDATORY),
2805         UVERBS_ATTR_PTR_IN(
2806                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2807                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2808                 UA_MANDATORY,
2809                 UA_ALLOC_AND_COPY),
2810         UVERBS_ATTR_PTR_OUT(
2811                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2812                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2813                 UA_MANDATORY));
2814
2815 DECLARE_UVERBS_NAMED_METHOD(
2816         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2817         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2818                         UVERBS_IDR_ANY_OBJECT,
2819                         UVERBS_ACCESS_READ,
2820                         UA_MANDATORY),
2821         UVERBS_ATTR_PTR_IN(
2822                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2823                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2824                 UA_MANDATORY,
2825                 UA_ALLOC_AND_COPY),
2826         UVERBS_ATTR_PTR_OUT(
2827                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2828                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2829                 UA_MANDATORY));
2830
2831 DECLARE_UVERBS_NAMED_METHOD(
2832         MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2833         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2834                         UVERBS_IDR_ANY_OBJECT,
2835                         UVERBS_ACCESS_READ,
2836                         UA_MANDATORY),
2837         UVERBS_ATTR_PTR_IN(
2838                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2839                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2840                 UA_MANDATORY,
2841                 UA_ALLOC_AND_COPY),
2842         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2843                 u16, UA_MANDATORY),
2844         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2845                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2846                 UVERBS_ACCESS_READ,
2847                 UA_MANDATORY),
2848         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2849                 UVERBS_ATTR_TYPE(u64),
2850                 UA_MANDATORY));
2851
2852 DECLARE_UVERBS_NAMED_METHOD(
2853         MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2854         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2855                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2856                 UVERBS_ACCESS_READ,
2857                 UA_MANDATORY),
2858         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2859                 MLX5_IB_OBJECT_DEVX_OBJ,
2860                 UVERBS_ACCESS_READ,
2861                 UA_OPTIONAL),
2862         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2863                 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2864                 UA_MANDATORY,
2865                 UA_ALLOC_AND_COPY),
2866         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2867                 UVERBS_ATTR_TYPE(u64),
2868                 UA_OPTIONAL),
2869         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2870                 UVERBS_ATTR_TYPE(u32),
2871                 UA_OPTIONAL));
2872
2873 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
2874                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2875                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
2876                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2877                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
2878
2879 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
2880                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2881                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2882                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2883                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
2884                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2885                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
2886
2887 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
2888                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2889                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2890                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
2891
2892
2893 DECLARE_UVERBS_NAMED_METHOD(
2894         MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2895         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2896                         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2897                         UVERBS_ACCESS_NEW,
2898                         UA_MANDATORY));
2899
2900 DECLARE_UVERBS_NAMED_OBJECT(
2901         MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2902         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
2903                              devx_hot_unplug_async_cmd_event_file,
2904                              &devx_async_cmd_event_fops, "[devx_async_cmd]",
2905                              O_RDONLY),
2906         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
2907
2908 DECLARE_UVERBS_NAMED_METHOD(
2909         MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
2910         UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
2911                         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2912                         UVERBS_ACCESS_NEW,
2913                         UA_MANDATORY),
2914         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
2915                         enum mlx5_ib_uapi_devx_create_event_channel_flags,
2916                         UA_MANDATORY));
2917
2918 DECLARE_UVERBS_NAMED_OBJECT(
2919         MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2920         UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
2921                              devx_hot_unplug_async_event_file,
2922                              &devx_async_event_fops, "[devx_async_event]",
2923                              O_RDONLY),
2924         &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
2925
2926 static bool devx_is_supported(struct ib_device *device)
2927 {
2928         struct mlx5_ib_dev *dev = to_mdev(device);
2929
2930         return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
2931 }
2932
2933 const struct uapi_definition mlx5_ib_devx_defs[] = {
2934         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2935                 MLX5_IB_OBJECT_DEVX,
2936                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2937         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2938                 MLX5_IB_OBJECT_DEVX_OBJ,
2939                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2940         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2941                 MLX5_IB_OBJECT_DEVX_UMEM,
2942                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2943         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2944                 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2945                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2946         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2947                 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2948                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2949         {},
2950 };