drivers/infiniband/core/nldev.c

   1 /*
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  * 3. Neither the names of the copyright holders nor the names of its
  13  *    contributors may be used to endorse or promote products derived from
  14  *    this software without specific prior written permission.
  15  *
  16  * Alternatively, this software may be distributed under the terms of the
  17  * GNU General Public License ("GPL") version 2 as published by the Free
  18  * Software Foundation.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <linux/module.h>
  34 #include <linux/pid.h>
  35 #include <linux/pid_namespace.h>
  36 #include <linux/mutex.h>
  37 #include <net/netlink.h>
  38 #include <rdma/rdma_cm.h>
  39 #include <rdma/rdma_netlink.h>
  40
  41 #include "core_priv.h"
  42 #include "cma_priv.h"
  43 #include "restrack.h"
  44 #include "uverbs.h"
  45
  46 typedef int (*res_fill_func_t)(struct sk_buff*, bool,
  47                                struct rdma_restrack_entry*, uint32_t);
  48
  49 /*
  50  * Sort array elements by the netlink attribute name
  51  */
  52 static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
  53         [RDMA_NLDEV_ATTR_CHARDEV]               = { .type = NLA_U64 },
  54         [RDMA_NLDEV_ATTR_CHARDEV_ABI]           = { .type = NLA_U64 },
  55         [RDMA_NLDEV_ATTR_CHARDEV_NAME]          = { .type = NLA_NUL_STRING,
  56                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  57         [RDMA_NLDEV_ATTR_CHARDEV_TYPE]          = { .type = NLA_NUL_STRING,
  58                                         .len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE },
  59         [RDMA_NLDEV_ATTR_DEV_DIM]               = { .type = NLA_U8 },
  60         [RDMA_NLDEV_ATTR_DEV_INDEX]             = { .type = NLA_U32 },
  61         [RDMA_NLDEV_ATTR_DEV_NAME]              = { .type = NLA_NUL_STRING,
  62                                         .len = IB_DEVICE_NAME_MAX },
  63         [RDMA_NLDEV_ATTR_DEV_NODE_TYPE]         = { .type = NLA_U8 },
  64         [RDMA_NLDEV_ATTR_DEV_PROTOCOL]          = { .type = NLA_NUL_STRING,
  65                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  66         [RDMA_NLDEV_ATTR_DRIVER]                = { .type = NLA_NESTED },
  67         [RDMA_NLDEV_ATTR_DRIVER_ENTRY]          = { .type = NLA_NESTED },
  68         [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]     = { .type = NLA_U8 },
  69         [RDMA_NLDEV_ATTR_DRIVER_STRING]         = { .type = NLA_NUL_STRING,
  70                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  71         [RDMA_NLDEV_ATTR_DRIVER_S32]            = { .type = NLA_S32 },
  72         [RDMA_NLDEV_ATTR_DRIVER_S64]            = { .type = NLA_S64 },
  73         [RDMA_NLDEV_ATTR_DRIVER_U32]            = { .type = NLA_U32 },
  74         [RDMA_NLDEV_ATTR_DRIVER_U64]            = { .type = NLA_U64 },
  75         [RDMA_NLDEV_ATTR_FW_VERSION]            = { .type = NLA_NUL_STRING,
  76                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  77         [RDMA_NLDEV_ATTR_LID]                   = { .type = NLA_U32 },
  78         [RDMA_NLDEV_ATTR_LINK_TYPE]             = { .type = NLA_NUL_STRING,
  79                                         .len = IFNAMSIZ },
  80         [RDMA_NLDEV_ATTR_LMC]                   = { .type = NLA_U8 },
  81         [RDMA_NLDEV_ATTR_NDEV_INDEX]            = { .type = NLA_U32 },
  82         [RDMA_NLDEV_ATTR_NDEV_NAME]             = { .type = NLA_NUL_STRING,
  83                                         .len = IFNAMSIZ },
  84         [RDMA_NLDEV_ATTR_NODE_GUID]             = { .type = NLA_U64 },
  85         [RDMA_NLDEV_ATTR_PORT_INDEX]            = { .type = NLA_U32 },
  86         [RDMA_NLDEV_ATTR_PORT_PHYS_STATE]       = { .type = NLA_U8 },
  87         [RDMA_NLDEV_ATTR_PORT_STATE]            = { .type = NLA_U8 },
  88         [RDMA_NLDEV_ATTR_RES_CM_ID]             = { .type = NLA_NESTED },
  89         [RDMA_NLDEV_ATTR_RES_CM_IDN]            = { .type = NLA_U32 },
  90         [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]       = { .type = NLA_NESTED },
  91         [RDMA_NLDEV_ATTR_RES_CQ]                = { .type = NLA_NESTED },
  92         [RDMA_NLDEV_ATTR_RES_CQE]               = { .type = NLA_U32 },
  93         [RDMA_NLDEV_ATTR_RES_CQN]               = { .type = NLA_U32 },
  94         [RDMA_NLDEV_ATTR_RES_CQ_ENTRY]          = { .type = NLA_NESTED },
  95         [RDMA_NLDEV_ATTR_RES_CTX]               = { .type = NLA_NESTED },
  96         [RDMA_NLDEV_ATTR_RES_CTXN]              = { .type = NLA_U32 },
  97         [RDMA_NLDEV_ATTR_RES_CTX_ENTRY]         = { .type = NLA_NESTED },
  98         [RDMA_NLDEV_ATTR_RES_DST_ADDR]          = {
  99                         .len = sizeof(struct __kernel_sockaddr_storage) },
 100         [RDMA_NLDEV_ATTR_RES_IOVA]              = { .type = NLA_U64 },
 101         [RDMA_NLDEV_ATTR_RES_KERN_NAME]         = { .type = NLA_NUL_STRING,
 102                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 103         [RDMA_NLDEV_ATTR_RES_LKEY]              = { .type = NLA_U32 },
 104         [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]    = { .type = NLA_U32 },
 105         [RDMA_NLDEV_ATTR_RES_LQPN]              = { .type = NLA_U32 },
 106         [RDMA_NLDEV_ATTR_RES_MR]                = { .type = NLA_NESTED },
 107         [RDMA_NLDEV_ATTR_RES_MRLEN]             = { .type = NLA_U64 },
 108         [RDMA_NLDEV_ATTR_RES_MRN]               = { .type = NLA_U32 },
 109         [RDMA_NLDEV_ATTR_RES_MR_ENTRY]          = { .type = NLA_NESTED },
 110         [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE]    = { .type = NLA_U8 },
 111         [RDMA_NLDEV_ATTR_RES_PD]                = { .type = NLA_NESTED },
 112         [RDMA_NLDEV_ATTR_RES_PDN]               = { .type = NLA_U32 },
 113         [RDMA_NLDEV_ATTR_RES_PD_ENTRY]          = { .type = NLA_NESTED },
 114         [RDMA_NLDEV_ATTR_RES_PID]               = { .type = NLA_U32 },
 115         [RDMA_NLDEV_ATTR_RES_POLL_CTX]          = { .type = NLA_U8 },
 116         [RDMA_NLDEV_ATTR_RES_PS]                = { .type = NLA_U32 },
 117         [RDMA_NLDEV_ATTR_RES_QP]                = { .type = NLA_NESTED },
 118         [RDMA_NLDEV_ATTR_RES_QP_ENTRY]          = { .type = NLA_NESTED },
 119         [RDMA_NLDEV_ATTR_RES_RAW]               = { .type = NLA_BINARY },
 120         [RDMA_NLDEV_ATTR_RES_RKEY]              = { .type = NLA_U32 },
 121         [RDMA_NLDEV_ATTR_RES_RQPN]              = { .type = NLA_U32 },
 122         [RDMA_NLDEV_ATTR_RES_RQ_PSN]            = { .type = NLA_U32 },
 123         [RDMA_NLDEV_ATTR_RES_SQ_PSN]            = { .type = NLA_U32 },
 124         [RDMA_NLDEV_ATTR_RES_SRC_ADDR]          = {
 125                         .len = sizeof(struct __kernel_sockaddr_storage) },
 126         [RDMA_NLDEV_ATTR_RES_STATE]             = { .type = NLA_U8 },
 127         [RDMA_NLDEV_ATTR_RES_SUMMARY]           = { .type = NLA_NESTED },
 128         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]     = { .type = NLA_NESTED },
 129         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 },
 130         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,
 131                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 132         [RDMA_NLDEV_ATTR_RES_TYPE]              = { .type = NLA_U8 },
 133         [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },
 134         [RDMA_NLDEV_ATTR_RES_USECNT]            = { .type = NLA_U64 },
 135         [RDMA_NLDEV_ATTR_RES_SRQ]               = { .type = NLA_NESTED },
 136         [RDMA_NLDEV_ATTR_RES_SRQN]              = { .type = NLA_U32 },
 137         [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY]         = { .type = NLA_NESTED },
 138         [RDMA_NLDEV_ATTR_MIN_RANGE]             = { .type = NLA_U32 },
 139         [RDMA_NLDEV_ATTR_MAX_RANGE]             = { .type = NLA_U32 },
 140         [RDMA_NLDEV_ATTR_SM_LID]                = { .type = NLA_U32 },
 141         [RDMA_NLDEV_ATTR_SUBNET_PREFIX]         = { .type = NLA_U64 },
 142         [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]   = { .type = NLA_U32 },
 143         [RDMA_NLDEV_ATTR_STAT_MODE]             = { .type = NLA_U32 },
 144         [RDMA_NLDEV_ATTR_STAT_RES]              = { .type = NLA_U32 },
 145         [RDMA_NLDEV_ATTR_STAT_COUNTER]          = { .type = NLA_NESTED },
 146         [RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY]    = { .type = NLA_NESTED },
 147         [RDMA_NLDEV_ATTR_STAT_COUNTER_ID]       = { .type = NLA_U32 },
 148         [RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]       = { .type = NLA_NESTED },
 149         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY]  = { .type = NLA_NESTED },
 150         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME] = { .type = NLA_NUL_STRING },
 151         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE] = { .type = NLA_U64 },
 152         [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID]        = { .type = NLA_U64 },
 153         [RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID]      = { .type = NLA_U32 },
 154         [RDMA_NLDEV_NET_NS_FD]                  = { .type = NLA_U32 },
 155         [RDMA_NLDEV_SYS_ATTR_NETNS_MODE]        = { .type = NLA_U8 },
 156         [RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]      = { .type = NLA_U8 },
 157 };
 158
 159 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
 160                                       enum rdma_nldev_print_type print_type)
 161 {
 162         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, name))
 163                 return -EMSGSIZE;
 164         if (print_type != RDMA_NLDEV_PRINT_TYPE_UNSPEC &&
 165             nla_put_u8(msg, RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE, print_type))
 166                 return -EMSGSIZE;
 167
 168         return 0;
 169 }
 170
 171 static int _rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name,
 172                                    enum rdma_nldev_print_type print_type,
 173                                    u32 value)
 174 {
 175         if (put_driver_name_print_type(msg, name, print_type))
 176                 return -EMSGSIZE;
 177         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DRIVER_U32, value))
 178                 return -EMSGSIZE;
 179
 180         return 0;
 181 }
 182
 183 static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name,
 184                                    enum rdma_nldev_print_type print_type,
 185                                    u64 value)
 186 {
 187         if (put_driver_name_print_type(msg, name, print_type))
 188                 return -EMSGSIZE;
 189         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_DRIVER_U64, value,
 190                               RDMA_NLDEV_ATTR_PAD))
 191                 return -EMSGSIZE;
 192
 193         return 0;
 194 }
 195
 196 int rdma_nl_put_driver_string(struct sk_buff *msg, const char *name,
 197                               const char *str)
 198 {
 199         if (put_driver_name_print_type(msg, name,
 200                                        RDMA_NLDEV_PRINT_TYPE_UNSPEC))
 201                 return -EMSGSIZE;
 202         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, str))
 203                 return -EMSGSIZE;
 204
 205         return 0;
 206 }
 207 EXPORT_SYMBOL(rdma_nl_put_driver_string);
 208
 209 int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value)
 210 {
 211         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 212                                        value);
 213 }
 214 EXPORT_SYMBOL(rdma_nl_put_driver_u32);
 215
 216 int rdma_nl_put_driver_u32_hex(struct sk_buff *msg, const char *name,
 217                                u32 value)
 218 {
 219         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 220                                        value);
 221 }
 222 EXPORT_SYMBOL(rdma_nl_put_driver_u32_hex);
 223
 224 int rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, u64 value)
 225 {
 226         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 227                                        value);
 228 }
 229 EXPORT_SYMBOL(rdma_nl_put_driver_u64);
 230
 231 int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value)
 232 {
 233         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 234                                        value);
 235 }
 236 EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);
 237
 238 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 239 {
 240         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
 241                 return -EMSGSIZE;
 242         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME,
 243                            dev_name(&device->dev)))
 244                 return -EMSGSIZE;
 245
 246         return 0;
 247 }
 248
 249 static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 250 {
 251         char fw[IB_FW_VERSION_NAME_MAX];
 252         int ret = 0;
 253         u32 port;
 254
 255         if (fill_nldev_handle(msg, device))
 256                 return -EMSGSIZE;
 257
 258         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device)))
 259                 return -EMSGSIZE;
 260
 261         BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64));
 262         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 263                               device->attrs.device_cap_flags,
 264                               RDMA_NLDEV_ATTR_PAD))
 265                 return -EMSGSIZE;
 266
 267         ib_get_device_fw_str(device, fw);
 268         /* Device without FW has strlen(fw) = 0 */
 269         if (strlen(fw) && nla_put_string(msg, RDMA_NLDEV_ATTR_FW_VERSION, fw))
 270                 return -EMSGSIZE;
 271
 272         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID,
 273                               be64_to_cpu(device->node_guid),
 274                               RDMA_NLDEV_ATTR_PAD))
 275                 return -EMSGSIZE;
 276         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID,
 277                               be64_to_cpu(device->attrs.sys_image_guid),
 278                               RDMA_NLDEV_ATTR_PAD))
 279                 return -EMSGSIZE;
 280         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
 281                 return -EMSGSIZE;
 282         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, device->use_cq_dim))
 283                 return -EMSGSIZE;
 284
 285         /*
 286          * Link type is determined on first port and mlx4 device
 287          * which can potentially have two different link type for the same
 288          * IB device is considered as better to be avoided in the future,
 289          */
 290         port = rdma_start_port(device);
 291         if (rdma_cap_opa_mad(device, port))
 292                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "opa");
 293         else if (rdma_protocol_ib(device, port))
 294                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "ib");
 295         else if (rdma_protocol_iwarp(device, port))
 296                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "iw");
 297         else if (rdma_protocol_roce(device, port))
 298                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "roce");
 299         else if (rdma_protocol_usnic(device, port))
 300                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL,
 301                                      "usnic");
 302         return ret;
 303 }
 304
 305 static int fill_port_info(struct sk_buff *msg,
 306                           struct ib_device *device, u32 port,
 307                           const struct net *net)
 308 {
 309         struct net_device *netdev = NULL;
 310         struct ib_port_attr attr;
 311         int ret;
 312         u64 cap_flags = 0;
 313
 314         if (fill_nldev_handle(msg, device))
 315                 return -EMSGSIZE;
 316
 317         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
 318                 return -EMSGSIZE;
 319
 320         ret = ib_query_port(device, port, &attr);
 321         if (ret)
 322                 return ret;
 323
 324         if (rdma_protocol_ib(device, port)) {
 325                 BUILD_BUG_ON((sizeof(attr.port_cap_flags) +
 326                                 sizeof(attr.port_cap_flags2)) > sizeof(u64));
 327                 cap_flags = attr.port_cap_flags |
 328                         ((u64)attr.port_cap_flags2 << 32);
 329                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 330                                       cap_flags, RDMA_NLDEV_ATTR_PAD))
 331                         return -EMSGSIZE;
 332                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX,
 333                                       attr.subnet_prefix, RDMA_NLDEV_ATTR_PAD))
 334                         return -EMSGSIZE;
 335                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid))
 336                         return -EMSGSIZE;
 337                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_SM_LID, attr.sm_lid))
 338                         return -EMSGSIZE;
 339                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_LMC, attr.lmc))
 340                         return -EMSGSIZE;
 341         }
 342         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_STATE, attr.state))
 343                 return -EMSGSIZE;
 344         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_PHYS_STATE, attr.phys_state))
 345                 return -EMSGSIZE;
 346
 347         netdev = ib_device_get_netdev(device, port);
 348         if (netdev && net_eq(dev_net(netdev), net)) {
 349                 ret = nla_put_u32(msg,
 350                                   RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex);
 351                 if (ret)
 352                         goto out;
 353                 ret = nla_put_string(msg,
 354                                      RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name);
 355         }
 356
 357 out:
 358         if (netdev)
 359                 dev_put(netdev);
 360         return ret;
 361 }
 362
 363 static int fill_res_info_entry(struct sk_buff *msg,
 364                                const char *name, u64 curr)
 365 {
 366         struct nlattr *entry_attr;
 367
 368         entry_attr = nla_nest_start_noflag(msg,
 369                                            RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
 370         if (!entry_attr)
 371                 return -EMSGSIZE;
 372
 373         if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
 374                 goto err;
 375         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr,
 376                               RDMA_NLDEV_ATTR_PAD))
 377                 goto err;
 378
 379         nla_nest_end(msg, entry_attr);
 380         return 0;
 381
 382 err:
 383         nla_nest_cancel(msg, entry_attr);
 384         return -EMSGSIZE;
 385 }
 386
 387 static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
 388 {
 389         static const char * const names[RDMA_RESTRACK_MAX] = {
 390                 [RDMA_RESTRACK_PD] = "pd",
 391                 [RDMA_RESTRACK_CQ] = "cq",
 392                 [RDMA_RESTRACK_QP] = "qp",
 393                 [RDMA_RESTRACK_CM_ID] = "cm_id",
 394                 [RDMA_RESTRACK_MR] = "mr",
 395                 [RDMA_RESTRACK_CTX] = "ctx",
 396                 [RDMA_RESTRACK_SRQ] = "srq",
 397         };
 398
 399         struct nlattr *table_attr;
 400         int ret, i, curr;
 401
 402         if (fill_nldev_handle(msg, device))
 403                 return -EMSGSIZE;
 404
 405         table_attr = nla_nest_start_noflag(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
 406         if (!table_attr)
 407                 return -EMSGSIZE;
 408
 409         for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
 410                 if (!names[i])
 411                         continue;
 412                 curr = rdma_restrack_count(device, i);
 413                 ret = fill_res_info_entry(msg, names[i], curr);
 414                 if (ret)
 415                         goto err;
 416         }
 417
 418         nla_nest_end(msg, table_attr);
 419         return 0;
 420
 421 err:
 422         nla_nest_cancel(msg, table_attr);
 423         return ret;
 424 }
 425
 426 static int fill_res_name_pid(struct sk_buff *msg,
 427                              struct rdma_restrack_entry *res)
 428 {
 429         int err = 0;
 430
 431         /*
 432          * For user resources, user is should read /proc/PID/comm to get the
 433          * name of the task file.
 434          */
 435         if (rdma_is_kernel_res(res)) {
 436                 err = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,
 437                                      res->kern_name);
 438         } else {
 439                 pid_t pid;
 440
 441                 pid = task_pid_vnr(res->task);
 442                 /*
 443                  * Task is dead and in zombie state.
 444                  * There is no need to print PID anymore.
 445                  */
 446                 if (pid)
 447                         /*
 448                          * This part is racy, task can be killed and PID will
 449                          * be zero right here but it is ok, next query won't
 450                          * return PID. We don't promise real-time reflection
 451                          * of SW objects.
 452                          */
 453                         err = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, pid);
 454         }
 455
 456         return err ? -EMSGSIZE : 0;
 457 }
 458
 459 static int fill_res_qp_entry_query(struct sk_buff *msg,
 460                                    struct rdma_restrack_entry *res,
 461                                    struct ib_device *dev,
 462                                    struct ib_qp *qp)
 463 {
 464         struct ib_qp_init_attr qp_init_attr;
 465         struct ib_qp_attr qp_attr;
 466         int ret;
 467
 468         ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr);
 469         if (ret)
 470                 return ret;
 471
 472         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
 473                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
 474                                 qp_attr.dest_qp_num))
 475                         goto err;
 476                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQ_PSN,
 477                                 qp_attr.rq_psn))
 478                         goto err;
 479         }
 480
 481         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SQ_PSN, qp_attr.sq_psn))
 482                 goto err;
 483
 484         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC ||
 485             qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) {
 486                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE,
 487                                qp_attr.path_mig_state))
 488                         goto err;
 489         }
 490         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type))
 491                 goto err;
 492         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
 493                 goto err;
 494
 495         if (dev->ops.fill_res_qp_entry)
 496                 return dev->ops.fill_res_qp_entry(msg, qp);
 497         return 0;
 498
 499 err:    return -EMSGSIZE;
 500 }
 501
 502 static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
 503                              struct rdma_restrack_entry *res, uint32_t port)
 504 {
 505         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 506         struct ib_device *dev = qp->device;
 507         int ret;
 508
 509         if (port && port != qp->port)
 510                 return -EAGAIN;
 511
 512         /* In create_qp() port is not set yet */
 513         if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
 514                 return -EINVAL;
 515
 516         ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
 517         if (ret)
 518                 return -EMSGSIZE;
 519
 520         if (!rdma_is_kernel_res(res) &&
 521             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
 522                 return -EMSGSIZE;
 523
 524         ret = fill_res_name_pid(msg, res);
 525         if (ret)
 526                 return -EMSGSIZE;
 527
 528         return fill_res_qp_entry_query(msg, res, dev, qp);
 529 }
 530
 531 static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 532                                  struct rdma_restrack_entry *res, uint32_t port)
 533 {
 534         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 535         struct ib_device *dev = qp->device;
 536
 537         if (port && port != qp->port)
 538                 return -EAGAIN;
 539         if (!dev->ops.fill_res_qp_entry_raw)
 540                 return -EINVAL;
 541         return dev->ops.fill_res_qp_entry_raw(msg, qp);
 542 }
 543
 544 static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,
 545                                 struct rdma_restrack_entry *res, uint32_t port)
 546 {
 547         struct rdma_id_private *id_priv =
 548                                 container_of(res, struct rdma_id_private, res);
 549         struct ib_device *dev = id_priv->id.device;
 550         struct rdma_cm_id *cm_id = &id_priv->id;
 551
 552         if (port && port != cm_id->port_num)
 553                 return 0;
 554
 555         if (cm_id->port_num &&
 556             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
 557                 goto err;
 558
 559         if (id_priv->qp_num) {
 560                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv->qp_num))
 561                         goto err;
 562                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type))
 563                         goto err;
 564         }
 565
 566         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps))
 567                 goto err;
 568
 569         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state))
 570                 goto err;
 571
 572         if (cm_id->route.addr.src_addr.ss_family &&
 573             nla_put(msg, RDMA_NLDEV_ATTR_RES_SRC_ADDR,
 574                     sizeof(cm_id->route.addr.src_addr),
 575                     &cm_id->route.addr.src_addr))
 576                 goto err;
 577         if (cm_id->route.addr.dst_addr.ss_family &&
 578             nla_put(msg, RDMA_NLDEV_ATTR_RES_DST_ADDR,
 579                     sizeof(cm_id->route.addr.dst_addr),
 580                     &cm_id->route.addr.dst_addr))
 581                 goto err;
 582
 583         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CM_IDN, res->id))
 584                 goto err;
 585
 586         if (fill_res_name_pid(msg, res))
 587                 goto err;
 588
 589         if (dev->ops.fill_res_cm_id_entry)
 590                 return dev->ops.fill_res_cm_id_entry(msg, cm_id);
 591         return 0;
 592
 593 err: return -EMSGSIZE;
 594 }
 595
 596 static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 597                              struct rdma_restrack_entry *res, uint32_t port)
 598 {
 599         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 600         struct ib_device *dev = cq->device;
 601
 602         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
 603                 return -EMSGSIZE;
 604         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 605                               atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
 606                 return -EMSGSIZE;
 607
 608         /* Poll context is only valid for kernel CQs */
 609         if (rdma_is_kernel_res(res) &&
 610             nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx))
 611                 return -EMSGSIZE;
 612
 613         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL)))
 614                 return -EMSGSIZE;
 615
 616         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
 617                 return -EMSGSIZE;
 618         if (!rdma_is_kernel_res(res) &&
 619             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 620                         cq->uobject->uevent.uobject.context->res.id))
 621                 return -EMSGSIZE;
 622
 623         if (fill_res_name_pid(msg, res))
 624                 return -EMSGSIZE;
 625
 626         return (dev->ops.fill_res_cq_entry) ?
 627                 dev->ops.fill_res_cq_entry(msg, cq) : 0;
 628 }
 629
 630 static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 631                                  struct rdma_restrack_entry *res, uint32_t port)
 632 {
 633         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 634         struct ib_device *dev = cq->device;
 635
 636         if (!dev->ops.fill_res_cq_entry_raw)
 637                 return -EINVAL;
 638         return dev->ops.fill_res_cq_entry_raw(msg, cq);
 639 }
 640
 641 static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 642                              struct rdma_restrack_entry *res, uint32_t port)
 643 {
 644         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 645         struct ib_device *dev = mr->pd->device;
 646
 647         if (has_cap_net_admin) {
 648                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
 649                         return -EMSGSIZE;
 650                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
 651                         return -EMSGSIZE;
 652         }
 653
 654         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
 655                               RDMA_NLDEV_ATTR_PAD))
 656                 return -EMSGSIZE;
 657
 658         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 659                 return -EMSGSIZE;
 660
 661         if (!rdma_is_kernel_res(res) &&
 662             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
 663                 return -EMSGSIZE;
 664
 665         if (fill_res_name_pid(msg, res))
 666                 return -EMSGSIZE;
 667
 668         return (dev->ops.fill_res_mr_entry) ?
 669                        dev->ops.fill_res_mr_entry(msg, mr) :
 670                        0;
 671 }
 672
 673 static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 674                                  struct rdma_restrack_entry *res, uint32_t port)
 675 {
 676         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 677         struct ib_device *dev = mr->pd->device;
 678
 679         if (!dev->ops.fill_res_mr_entry_raw)
 680                 return -EINVAL;
 681         return dev->ops.fill_res_mr_entry_raw(msg, mr);
 682 }
 683
 684 static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 685                              struct rdma_restrack_entry *res, uint32_t port)
 686 {
 687         struct ib_pd *pd = container_of(res, struct ib_pd, res);
 688
 689         if (has_cap_net_admin) {
 690                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,
 691                                 pd->local_dma_lkey))
 692                         goto err;
 693                 if ((pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) &&
 694                     nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,
 695                                 pd->unsafe_global_rkey))
 696                         goto err;
 697         }
 698         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 699                               atomic_read(&pd->usecnt), RDMA_NLDEV_ATTR_PAD))
 700                 goto err;
 701
 702         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
 703                 goto err;
 704
 705         if (!rdma_is_kernel_res(res) &&
 706             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 707                         pd->uobject->context->res.id))
 708                 goto err;
 709
 710         return fill_res_name_pid(msg, res);
 711
 712 err:    return -EMSGSIZE;
 713 }
 714
 715 static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,
 716                               struct rdma_restrack_entry *res, uint32_t port)
 717 {
 718         struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res);
 719
 720         if (rdma_is_kernel_res(res))
 721                 return 0;
 722
 723         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, ctx->res.id))
 724                 return -EMSGSIZE;
 725
 726         return fill_res_name_pid(msg, res);
 727 }
 728
 729 static int fill_res_range_qp_entry(struct sk_buff *msg, uint32_t min_range,
 730                                    uint32_t max_range)
 731 {
 732         struct nlattr *entry_attr;
 733
 734         if (!min_range)
 735                 return 0;
 736
 737         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 738         if (!entry_attr)
 739                 return -EMSGSIZE;
 740
 741         if (min_range == max_range) {
 742                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, min_range))
 743                         goto err;
 744         } else {
 745                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MIN_RANGE, min_range))
 746                         goto err;
 747                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MAX_RANGE, max_range))
 748                         goto err;
 749         }
 750         nla_nest_end(msg, entry_attr);
 751         return 0;
 752
 753 err:
 754         nla_nest_cancel(msg, entry_attr);
 755         return -EMSGSIZE;
 756 }
 757
 758 static int fill_res_srq_qps(struct sk_buff *msg, struct ib_srq *srq)
 759 {
 760         uint32_t min_range = 0, prev = 0;
 761         struct rdma_restrack_entry *res;
 762         struct rdma_restrack_root *rt;
 763         struct nlattr *table_attr;
 764         struct ib_qp *qp = NULL;
 765         unsigned long id = 0;
 766
 767         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 768         if (!table_attr)
 769                 return -EMSGSIZE;
 770
 771         rt = &srq->device->res[RDMA_RESTRACK_QP];
 772         xa_lock(&rt->xa);
 773         xa_for_each(&rt->xa, id, res) {
 774                 if (!rdma_restrack_get(res))
 775                         continue;
 776
 777                 qp = container_of(res, struct ib_qp, res);
 778                 if (!qp->srq || (qp->srq->res.id != srq->res.id)) {
 779                         rdma_restrack_put(res);
 780                         continue;
 781                 }
 782
 783                 if (qp->qp_num < prev)
 784                         /* qp_num should be ascending */
 785                         goto err_loop;
 786
 787                 if (min_range == 0) {
 788                         min_range = qp->qp_num;
 789                 } else if (qp->qp_num > (prev + 1)) {
 790                         if (fill_res_range_qp_entry(msg, min_range, prev))
 791                                 goto err_loop;
 792
 793                         min_range = qp->qp_num;
 794                 }
 795                 prev = qp->qp_num;
 796                 rdma_restrack_put(res);
 797         }
 798
 799         xa_unlock(&rt->xa);
 800
 801         if (fill_res_range_qp_entry(msg, min_range, prev))
 802                 goto err;
 803
 804         nla_nest_end(msg, table_attr);
 805         return 0;
 806
 807 err_loop:
 808         rdma_restrack_put(res);
 809         xa_unlock(&rt->xa);
 810 err:
 811         nla_nest_cancel(msg, table_attr);
 812         return -EMSGSIZE;
 813 }
 814
 815 static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 816                               struct rdma_restrack_entry *res, uint32_t port)
 817 {
 818         struct ib_srq *srq = container_of(res, struct ib_srq, res);
 819
 820         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SRQN, srq->res.id))
 821                 goto err;
 822
 823         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, srq->srq_type))
 824                 goto err;
 825
 826         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, srq->pd->res.id))
 827                 goto err;
 828
 829         if (ib_srq_has_cq(srq->srq_type)) {
 830                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN,
 831                                 srq->ext.cq->res.id))
 832                         goto err;
 833         }
 834
 835         if (fill_res_srq_qps(msg, srq))
 836                 goto err;
 837
 838         return fill_res_name_pid(msg, res);
 839
 840 err:
 841         return -EMSGSIZE;
 842 }
 843
 844 static int fill_stat_counter_mode(struct sk_buff *msg,
 845                                   struct rdma_counter *counter)
 846 {
 847         struct rdma_counter_mode *m = &counter->mode;
 848
 849         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode))
 850                 return -EMSGSIZE;
 851
 852         if (m->mode == RDMA_COUNTER_MODE_AUTO) {
 853                 if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) &&
 854                     nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type))
 855                         return -EMSGSIZE;
 856
 857                 if ((m->mask & RDMA_COUNTER_MASK_PID) &&
 858                     fill_res_name_pid(msg, &counter->res))
 859                         return -EMSGSIZE;
 860         }
 861
 862         return 0;
 863 }
 864
 865 static int fill_stat_counter_qp_entry(struct sk_buff *msg, u32 qpn)
 866 {
 867         struct nlattr *entry_attr;
 868
 869         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 870         if (!entry_attr)
 871                 return -EMSGSIZE;
 872
 873         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn))
 874                 goto err;
 875
 876         nla_nest_end(msg, entry_attr);
 877         return 0;
 878
 879 err:
 880         nla_nest_cancel(msg, entry_attr);
 881         return -EMSGSIZE;
 882 }
 883
 884 static int fill_stat_counter_qps(struct sk_buff *msg,
 885                                  struct rdma_counter *counter)
 886 {
 887         struct rdma_restrack_entry *res;
 888         struct rdma_restrack_root *rt;
 889         struct nlattr *table_attr;
 890         struct ib_qp *qp = NULL;
 891         unsigned long id = 0;
 892         int ret = 0;
 893
 894         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 895
 896         rt = &counter->device->res[RDMA_RESTRACK_QP];
 897         xa_lock(&rt->xa);
 898         xa_for_each(&rt->xa, id, res) {
 899                 qp = container_of(res, struct ib_qp, res);
 900                 if (!qp->counter || (qp->counter->id != counter->id))
 901                         continue;
 902
 903                 ret = fill_stat_counter_qp_entry(msg, qp->qp_num);
 904                 if (ret)
 905                         goto err;
 906         }
 907
 908         xa_unlock(&rt->xa);
 909         nla_nest_end(msg, table_attr);
 910         return 0;
 911
 912 err:
 913         xa_unlock(&rt->xa);
 914         nla_nest_cancel(msg, table_attr);
 915         return ret;
 916 }
 917
 918 int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name,
 919                                  u64 value)
 920 {
 921         struct nlattr *entry_attr;
 922
 923         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);
 924         if (!entry_attr)
 925                 return -EMSGSIZE;
 926
 927         if (nla_put_string(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,
 928                            name))
 929                 goto err;
 930         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE,
 931                               value, RDMA_NLDEV_ATTR_PAD))
 932                 goto err;
 933
 934         nla_nest_end(msg, entry_attr);
 935         return 0;
 936
 937 err:
 938         nla_nest_cancel(msg, entry_attr);
 939         return -EMSGSIZE;
 940 }
 941 EXPORT_SYMBOL(rdma_nl_stat_hwcounter_entry);
 942
 943 static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 944                               struct rdma_restrack_entry *res, uint32_t port)
 945 {
 946         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 947         struct ib_device *dev = mr->pd->device;
 948
 949         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 950                 goto err;
 951
 952         if (dev->ops.fill_stat_mr_entry)
 953                 return dev->ops.fill_stat_mr_entry(msg, mr);
 954         return 0;
 955
 956 err:
 957         return -EMSGSIZE;
 958 }
 959
 960 static int fill_stat_counter_hwcounters(struct sk_buff *msg,
 961                                         struct rdma_counter *counter)
 962 {
 963         struct rdma_hw_stats *st = counter->stats;
 964         struct nlattr *table_attr;
 965         int i;
 966
 967         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
 968         if (!table_attr)
 969                 return -EMSGSIZE;
 970
 971         for (i = 0; i < st->num_counters; i++)
 972                 if (rdma_nl_stat_hwcounter_entry(msg, st->names[i], st->value[i]))
 973                         goto err;
 974
 975         nla_nest_end(msg, table_attr);
 976         return 0;
 977
 978 err:
 979         nla_nest_cancel(msg, table_attr);
 980         return -EMSGSIZE;
 981 }
 982
 983 static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin,
 984                                   struct rdma_restrack_entry *res,
 985                                   uint32_t port)
 986 {
 987         struct rdma_counter *counter =
 988                 container_of(res, struct rdma_counter, res);
 989
 990         if (port && port != counter->port)
 991                 return -EAGAIN;
 992
 993         /* Dump it even query failed */
 994         rdma_counter_query_stats(counter);
 995
 996         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) ||
 997             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) ||
 998             fill_stat_counter_mode(msg, counter) ||
 999             fill_stat_counter_qps(msg, counter) ||
1000             fill_stat_counter_hwcounters(msg, counter))
1001                 return -EMSGSIZE;
1002
1003         return 0;
1004 }
1005
1006 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1007                           struct netlink_ext_ack *extack)
1008 {
1009         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1010         struct ib_device *device;
1011         struct sk_buff *msg;
1012         u32 index;
1013         int err;
1014
1015         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1016                                      nldev_policy, extack);
1017         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1018                 return -EINVAL;
1019
1020         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1021
1022         device = ib_device_get_by_index(sock_net(skb->sk), index);
1023         if (!device)
1024                 return -EINVAL;
1025
1026         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1027         if (!msg) {
1028                 err = -ENOMEM;
1029                 goto err;
1030         }
1031
1032         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1033                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1034                         0, 0);
1035
1036         err = fill_dev_info(msg, device);
1037         if (err)
1038                 goto err_free;
1039
1040         nlmsg_end(msg, nlh);
1041
1042         ib_device_put(device);
1043         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1044
1045 err_free:
1046         nlmsg_free(msg);
1047 err:
1048         ib_device_put(device);
1049         return err;
1050 }
1051
1052 static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1053                           struct netlink_ext_ack *extack)
1054 {
1055         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1056         struct ib_device *device;
1057         u32 index;
1058         int err;
1059
1060         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1061                                      nldev_policy, extack);
1062         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1063                 return -EINVAL;
1064
1065         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1066         device = ib_device_get_by_index(sock_net(skb->sk), index);
1067         if (!device)
1068                 return -EINVAL;
1069
1070         if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) {
1071                 char name[IB_DEVICE_NAME_MAX] = {};
1072
1073                 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1074                             IB_DEVICE_NAME_MAX);
1075                 if (strlen(name) == 0) {
1076                         err = -EINVAL;
1077                         goto done;
1078                 }
1079                 err = ib_device_rename(device, name);
1080                 goto done;
1081         }
1082
1083         if (tb[RDMA_NLDEV_NET_NS_FD]) {
1084                 u32 ns_fd;
1085
1086                 ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]);
1087                 err = ib_device_set_netns_put(skb, device, ns_fd);
1088                 goto put_done;
1089         }
1090
1091         if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) {
1092                 u8 use_dim;
1093
1094                 use_dim = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]);
1095                 err = ib_device_set_dim(device,  use_dim);
1096                 goto done;
1097         }
1098
1099 done:
1100         ib_device_put(device);
1101 put_done:
1102         return err;
1103 }
1104
1105 static int _nldev_get_dumpit(struct ib_device *device,
1106                              struct sk_buff *skb,
1107                              struct netlink_callback *cb,
1108                              unsigned int idx)
1109 {
1110         int start = cb->args[0];
1111         struct nlmsghdr *nlh;
1112
1113         if (idx < start)
1114                 return 0;
1115
1116         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1117                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1118                         0, NLM_F_MULTI);
1119
1120         if (fill_dev_info(skb, device)) {
1121                 nlmsg_cancel(skb, nlh);
1122                 goto out;
1123         }
1124
1125         nlmsg_end(skb, nlh);
1126
1127         idx++;
1128
1129 out:    cb->args[0] = idx;
1130         return skb->len;
1131 }
1132
1133 static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
1134 {
1135         /*
1136          * There is no need to take lock, because
1137          * we are relying on ib_core's locking.
1138          */
1139         return ib_enum_all_devs(_nldev_get_dumpit, skb, cb);
1140 }
1141
1142 static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1143                                struct netlink_ext_ack *extack)
1144 {
1145         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1146         struct ib_device *device;
1147         struct sk_buff *msg;
1148         u32 index;
1149         u32 port;
1150         int err;
1151
1152         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1153                                      nldev_policy, extack);
1154         if (err ||
1155             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
1156             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
1157                 return -EINVAL;
1158
1159         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1160         device = ib_device_get_by_index(sock_net(skb->sk), index);
1161         if (!device)
1162                 return -EINVAL;
1163
1164         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1165         if (!rdma_is_port_valid(device, port)) {
1166                 err = -EINVAL;
1167                 goto err;
1168         }
1169
1170         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1171         if (!msg) {
1172                 err = -ENOMEM;
1173                 goto err;
1174         }
1175
1176         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1177                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1178                         0, 0);
1179
1180         err = fill_port_info(msg, device, port, sock_net(skb->sk));
1181         if (err)
1182                 goto err_free;
1183
1184         nlmsg_end(msg, nlh);
1185         ib_device_put(device);
1186
1187         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1188
1189 err_free:
1190         nlmsg_free(msg);
1191 err:
1192         ib_device_put(device);
1193         return err;
1194 }
1195
1196 static int nldev_port_get_dumpit(struct sk_buff *skb,
1197                                  struct netlink_callback *cb)
1198 {
1199         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1200         struct ib_device *device;
1201         int start = cb->args[0];
1202         struct nlmsghdr *nlh;
1203         u32 idx = 0;
1204         u32 ifindex;
1205         int err;
1206         unsigned int p;
1207
1208         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1209                                      nldev_policy, NULL);
1210         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1211                 return -EINVAL;
1212
1213         ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1214         device = ib_device_get_by_index(sock_net(skb->sk), ifindex);
1215         if (!device)
1216                 return -EINVAL;
1217
1218         rdma_for_each_port (device, p) {
1219                 /*
1220                  * The dumpit function returns all information from specific
1221                  * index. This specific index is taken from the netlink
1222                  * messages request sent by user and it is available
1223                  * in cb->args[0].
1224                  *
1225                  * Usually, the user doesn't fill this field and it causes
1226                  * to return everything.
1227                  *
1228                  */
1229                 if (idx < start) {
1230                         idx++;
1231                         continue;
1232                 }
1233
1234                 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
1235                                 cb->nlh->nlmsg_seq,
1236                                 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1237                                                  RDMA_NLDEV_CMD_PORT_GET),
1238                                 0, NLM_F_MULTI);
1239
1240                 if (fill_port_info(skb, device, p, sock_net(skb->sk))) {
1241                         nlmsg_cancel(skb, nlh);
1242                         goto out;
1243                 }
1244                 idx++;
1245                 nlmsg_end(skb, nlh);
1246         }
1247
1248 out:
1249         ib_device_put(device);
1250         cb->args[0] = idx;
1251         return skb->len;
1252 }
1253
1254 static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1255                               struct netlink_ext_ack *extack)
1256 {
1257         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1258         struct ib_device *device;
1259         struct sk_buff *msg;
1260         u32 index;
1261         int ret;
1262
1263         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1264                                      nldev_policy, extack);
1265         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1266                 return -EINVAL;
1267
1268         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1269         device = ib_device_get_by_index(sock_net(skb->sk), index);
1270         if (!device)
1271                 return -EINVAL;
1272
1273         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1274         if (!msg) {
1275                 ret = -ENOMEM;
1276                 goto err;
1277         }
1278
1279         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1280                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1281                         0, 0);
1282
1283         ret = fill_res_info(msg, device);
1284         if (ret)
1285                 goto err_free;
1286
1287         nlmsg_end(msg, nlh);
1288         ib_device_put(device);
1289         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1290
1291 err_free:
1292         nlmsg_free(msg);
1293 err:
1294         ib_device_put(device);
1295         return ret;
1296 }
1297
1298 static int _nldev_res_get_dumpit(struct ib_device *device,
1299                                  struct sk_buff *skb,
1300                                  struct netlink_callback *cb,
1301                                  unsigned int idx)
1302 {
1303         int start = cb->args[0];
1304         struct nlmsghdr *nlh;
1305
1306         if (idx < start)
1307                 return 0;
1308
1309         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1310                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1311                         0, NLM_F_MULTI);
1312
1313         if (fill_res_info(skb, device)) {
1314                 nlmsg_cancel(skb, nlh);
1315                 goto out;
1316         }
1317         nlmsg_end(skb, nlh);
1318
1319         idx++;
1320
1321 out:
1322         cb->args[0] = idx;
1323         return skb->len;
1324 }
1325
1326 static int nldev_res_get_dumpit(struct sk_buff *skb,
1327                                 struct netlink_callback *cb)
1328 {
1329         return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
1330 }
1331
1332 struct nldev_fill_res_entry {
1333         enum rdma_nldev_attr nldev_attr;
1334         u8 flags;
1335         u32 entry;
1336         u32 id;
1337 };
1338
1339 enum nldev_res_flags {
1340         NLDEV_PER_DEV = 1 << 0,
1341 };
1342
1343 static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
1344         [RDMA_RESTRACK_QP] = {
1345                 .nldev_attr = RDMA_NLDEV_ATTR_RES_QP,
1346                 .entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,
1347                 .id = RDMA_NLDEV_ATTR_RES_LQPN,
1348         },
1349         [RDMA_RESTRACK_CM_ID] = {
1350                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,
1351                 .entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,
1352                 .id = RDMA_NLDEV_ATTR_RES_CM_IDN,
1353         },
1354         [RDMA_RESTRACK_CQ] = {
1355                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
1356                 .flags = NLDEV_PER_DEV,
1357                 .entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,
1358                 .id = RDMA_NLDEV_ATTR_RES_CQN,
1359         },
1360         [RDMA_RESTRACK_MR] = {
1361                 .nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
1362                 .flags = NLDEV_PER_DEV,
1363                 .entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,
1364                 .id = RDMA_NLDEV_ATTR_RES_MRN,
1365         },
1366         [RDMA_RESTRACK_PD] = {
1367                 .nldev_attr = RDMA_NLDEV_ATTR_RES_PD,
1368                 .flags = NLDEV_PER_DEV,
1369                 .entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,
1370                 .id = RDMA_NLDEV_ATTR_RES_PDN,
1371         },
1372         [RDMA_RESTRACK_COUNTER] = {
1373                 .nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,
1374                 .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,
1375                 .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,
1376         },
1377         [RDMA_RESTRACK_CTX] = {
1378                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CTX,
1379                 .flags = NLDEV_PER_DEV,
1380                 .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY,
1381                 .id = RDMA_NLDEV_ATTR_RES_CTXN,
1382         },
1383         [RDMA_RESTRACK_SRQ] = {
1384                 .nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ,
1385                 .flags = NLDEV_PER_DEV,
1386                 .entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY,
1387                 .id = RDMA_NLDEV_ATTR_RES_SRQN,
1388         },
1389
1390 };
1391
1392 static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1393                                struct netlink_ext_ack *extack,
1394                                enum rdma_restrack_type res_type,
1395                                res_fill_func_t fill_func)
1396 {
1397         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1398         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1399         struct rdma_restrack_entry *res;
1400         struct ib_device *device;
1401         u32 index, id, port = 0;
1402         bool has_cap_net_admin;
1403         struct sk_buff *msg;
1404         int ret;
1405
1406         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1407                                      nldev_policy, extack);
1408         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !fe->id || !tb[fe->id])
1409                 return -EINVAL;
1410
1411         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1412         device = ib_device_get_by_index(sock_net(skb->sk), index);
1413         if (!device)
1414                 return -EINVAL;
1415
1416         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1417                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1418                 if (!rdma_is_port_valid(device, port)) {
1419                         ret = -EINVAL;
1420                         goto err;
1421                 }
1422         }
1423
1424         if ((port && fe->flags & NLDEV_PER_DEV) ||
1425             (!port && ~fe->flags & NLDEV_PER_DEV)) {
1426                 ret = -EINVAL;
1427                 goto err;
1428         }
1429
1430         id = nla_get_u32(tb[fe->id]);
1431         res = rdma_restrack_get_byid(device, res_type, id);
1432         if (IS_ERR(res)) {
1433                 ret = PTR_ERR(res);
1434                 goto err;
1435         }
1436
1437         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1438         if (!msg) {
1439                 ret = -ENOMEM;
1440                 goto err_get;
1441         }
1442
1443         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1444                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1445                                          RDMA_NL_GET_OP(nlh->nlmsg_type)),
1446                         0, 0);
1447
1448         if (fill_nldev_handle(msg, device)) {
1449                 ret = -EMSGSIZE;
1450                 goto err_free;
1451         }
1452
1453         has_cap_net_admin = netlink_capable(skb, CAP_NET_ADMIN);
1454
1455         ret = fill_func(msg, has_cap_net_admin, res, port);
1456         if (ret)
1457                 goto err_free;
1458
1459         rdma_restrack_put(res);
1460         nlmsg_end(msg, nlh);
1461         ib_device_put(device);
1462         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1463
1464 err_free:
1465         nlmsg_free(msg);
1466 err_get:
1467         rdma_restrack_put(res);
1468 err:
1469         ib_device_put(device);
1470         return ret;
1471 }
1472
1473 static int res_get_common_dumpit(struct sk_buff *skb,
1474                                  struct netlink_callback *cb,
1475                                  enum rdma_restrack_type res_type,
1476                                  res_fill_func_t fill_func)
1477 {
1478         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1479         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1480         struct rdma_restrack_entry *res;
1481         struct rdma_restrack_root *rt;
1482         int err, ret = 0, idx = 0;
1483         struct nlattr *table_attr;
1484         struct nlattr *entry_attr;
1485         struct ib_device *device;
1486         int start = cb->args[0];
1487         bool has_cap_net_admin;
1488         struct nlmsghdr *nlh;
1489         unsigned long id;
1490         u32 index, port = 0;
1491         bool filled = false;
1492
1493         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1494                                      nldev_policy, NULL);
1495         /*
1496          * Right now, we are expecting the device index to get res information,
1497          * but it is possible to extend this code to return all devices in
1498          * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX.
1499          * if it doesn't exist, we will iterate over all devices.
1500          *
1501          * But it is not needed for now.
1502          */
1503         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1504                 return -EINVAL;
1505
1506         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1507         device = ib_device_get_by_index(sock_net(skb->sk), index);
1508         if (!device)
1509                 return -EINVAL;
1510
1511         /*
1512          * If no PORT_INDEX is supplied, we will return all QPs from that device
1513          */
1514         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1515                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1516                 if (!rdma_is_port_valid(device, port)) {
1517                         ret = -EINVAL;
1518                         goto err_index;
1519                 }
1520         }
1521
1522         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1523                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1524                                          RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
1525                         0, NLM_F_MULTI);
1526
1527         if (fill_nldev_handle(skb, device)) {
1528                 ret = -EMSGSIZE;
1529                 goto err;
1530         }
1531
1532         table_attr = nla_nest_start_noflag(skb, fe->nldev_attr);
1533         if (!table_attr) {
1534                 ret = -EMSGSIZE;
1535                 goto err;
1536         }
1537
1538         has_cap_net_admin = netlink_capable(cb->skb, CAP_NET_ADMIN);
1539
1540         rt = &device->res[res_type];
1541         xa_lock(&rt->xa);
1542         /*
1543          * FIXME: if the skip ahead is something common this loop should
1544          * use xas_for_each & xas_pause to optimize, we can have a lot of
1545          * objects.
1546          */
1547         xa_for_each(&rt->xa, id, res) {
1548                 if (idx < start || !rdma_restrack_get(res))
1549                         goto next;
1550
1551                 xa_unlock(&rt->xa);
1552
1553                 filled = true;
1554
1555                 entry_attr = nla_nest_start_noflag(skb, fe->entry);
1556                 if (!entry_attr) {
1557                         ret = -EMSGSIZE;
1558                         rdma_restrack_put(res);
1559                         goto msg_full;
1560                 }
1561
1562                 ret = fill_func(skb, has_cap_net_admin, res, port);
1563
1564                 rdma_restrack_put(res);
1565
1566                 if (ret) {
1567                         nla_nest_cancel(skb, entry_attr);
1568                         if (ret == -EMSGSIZE)
1569                                 goto msg_full;
1570                         if (ret == -EAGAIN)
1571                                 goto again;
1572                         goto res_err;
1573                 }
1574                 nla_nest_end(skb, entry_attr);
1575 again:          xa_lock(&rt->xa);
1576 next:           idx++;
1577         }
1578         xa_unlock(&rt->xa);
1579
1580 msg_full:
1581         nla_nest_end(skb, table_attr);
1582         nlmsg_end(skb, nlh);
1583         cb->args[0] = idx;
1584
1585         /*
1586          * No more entries to fill, cancel the message and
1587          * return 0 to mark end of dumpit.
1588          */
1589         if (!filled)
1590                 goto err;
1591
1592         ib_device_put(device);
1593         return skb->len;
1594
1595 res_err:
1596         nla_nest_cancel(skb, table_attr);
1597
1598 err:
1599         nlmsg_cancel(skb, nlh);
1600
1601 err_index:
1602         ib_device_put(device);
1603         return ret;
1604 }
1605
1606 #define RES_GET_FUNCS(name, type)                                              \
1607         static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,          \
1608                                                  struct netlink_callback *cb)  \
1609         {                                                                      \
1610                 return res_get_common_dumpit(skb, cb, type,                    \
1611                                              fill_res_##name##_entry);         \
1612         }                                                                      \
1613         static int nldev_res_get_##name##_doit(struct sk_buff *skb,            \
1614                                                struct nlmsghdr *nlh,           \
1615                                                struct netlink_ext_ack *extack) \
1616         {                                                                      \
1617                 return res_get_common_doit(skb, nlh, extack, type,             \
1618                                            fill_res_##name##_entry);           \
1619         }
1620
1621 RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);
1622 RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);
1623 RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);
1624 RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);
1625 RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);
1626 RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);
1627 RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);
1628 RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);
1629 RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);
1630 RES_GET_FUNCS(ctx, RDMA_RESTRACK_CTX);
1631 RES_GET_FUNCS(srq, RDMA_RESTRACK_SRQ);
1632
1633 static LIST_HEAD(link_ops);
1634 static DECLARE_RWSEM(link_ops_rwsem);
1635
1636 static const struct rdma_link_ops *link_ops_get(const char *type)
1637 {
1638         const struct rdma_link_ops *ops;
1639
1640         list_for_each_entry(ops, &link_ops, list) {
1641                 if (!strcmp(ops->type, type))
1642                         goto out;
1643         }
1644         ops = NULL;
1645 out:
1646         return ops;
1647 }
1648
1649 void rdma_link_register(struct rdma_link_ops *ops)
1650 {
1651         down_write(&link_ops_rwsem);
1652         if (WARN_ON_ONCE(link_ops_get(ops->type)))
1653                 goto out;
1654         list_add(&ops->list, &link_ops);
1655 out:
1656         up_write(&link_ops_rwsem);
1657 }
1658 EXPORT_SYMBOL(rdma_link_register);
1659
1660 void rdma_link_unregister(struct rdma_link_ops *ops)
1661 {
1662         down_write(&link_ops_rwsem);
1663         list_del(&ops->list);
1664         up_write(&link_ops_rwsem);
1665 }
1666 EXPORT_SYMBOL(rdma_link_unregister);
1667
1668 static int nldev_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
1669                           struct netlink_ext_ack *extack)
1670 {
1671         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1672         char ibdev_name[IB_DEVICE_NAME_MAX];
1673         const struct rdma_link_ops *ops;
1674         char ndev_name[IFNAMSIZ];
1675         struct net_device *ndev;
1676         char type[IFNAMSIZ];
1677         int err;
1678
1679         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1680                                      nldev_policy, extack);
1681         if (err || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
1682             !tb[RDMA_NLDEV_ATTR_LINK_TYPE] || !tb[RDMA_NLDEV_ATTR_NDEV_NAME])
1683                 return -EINVAL;
1684
1685         nla_strscpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1686                     sizeof(ibdev_name));
1687         if (strchr(ibdev_name, '%') || strlen(ibdev_name) == 0)
1688                 return -EINVAL;
1689
1690         nla_strscpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type));
1691         nla_strscpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME],
1692                     sizeof(ndev_name));
1693
1694         ndev = dev_get_by_name(sock_net(skb->sk), ndev_name);
1695         if (!ndev)
1696                 return -ENODEV;
1697
1698         down_read(&link_ops_rwsem);
1699         ops = link_ops_get(type);
1700 #ifdef CONFIG_MODULES
1701         if (!ops) {
1702                 up_read(&link_ops_rwsem);
1703                 request_module("rdma-link-%s", type);
1704                 down_read(&link_ops_rwsem);
1705                 ops = link_ops_get(type);
1706         }
1707 #endif
1708         err = ops ? ops->newlink(ibdev_name, ndev) : -EINVAL;
1709         up_read(&link_ops_rwsem);
1710         dev_put(ndev);
1711
1712         return err;
1713 }
1714
1715 static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
1716                           struct netlink_ext_ack *extack)
1717 {
1718         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1719         struct ib_device *device;
1720         u32 index;
1721         int err;
1722
1723         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1724                                      nldev_policy, extack);
1725         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1726                 return -EINVAL;
1727
1728         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1729         device = ib_device_get_by_index(sock_net(skb->sk), index);
1730         if (!device)
1731                 return -EINVAL;
1732
1733         if (!(device->attrs.device_cap_flags & IB_DEVICE_ALLOW_USER_UNREG)) {
1734                 ib_device_put(device);
1735                 return -EINVAL;
1736         }
1737
1738         ib_unregister_device_and_put(device);
1739         return 0;
1740 }
1741
1742 static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh,
1743                              struct netlink_ext_ack *extack)
1744 {
1745         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1746         char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE];
1747         struct ib_client_nl_info data = {};
1748         struct ib_device *ibdev = NULL;
1749         struct sk_buff *msg;
1750         u32 index;
1751         int err;
1752
1753         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
1754                           extack);
1755         if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE])
1756                 return -EINVAL;
1757
1758         nla_strscpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE],
1759                     sizeof(client_name));
1760
1761         if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {
1762                 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1763                 ibdev = ib_device_get_by_index(sock_net(skb->sk), index);
1764                 if (!ibdev)
1765                         return -EINVAL;
1766
1767                 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1768                         data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1769                         if (!rdma_is_port_valid(ibdev, data.port)) {
1770                                 err = -EINVAL;
1771                                 goto out_put;
1772                         }
1773                 } else {
1774                         data.port = -1;
1775                 }
1776         } else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1777                 return -EINVAL;
1778         }
1779
1780         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1781         if (!msg) {
1782                 err = -ENOMEM;
1783                 goto out_put;
1784         }
1785         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1786                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1787                                          RDMA_NLDEV_CMD_GET_CHARDEV),
1788                         0, 0);
1789
1790         data.nl_msg = msg;
1791         err = ib_get_client_nl_info(ibdev, client_name, &data);
1792         if (err)
1793                 goto out_nlmsg;
1794
1795         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV,
1796                                 huge_encode_dev(data.cdev->devt),
1797                                 RDMA_NLDEV_ATTR_PAD);
1798         if (err)
1799                 goto out_data;
1800         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi,
1801                                 RDMA_NLDEV_ATTR_PAD);
1802         if (err)
1803                 goto out_data;
1804         if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME,
1805                            dev_name(data.cdev))) {
1806                 err = -EMSGSIZE;
1807                 goto out_data;
1808         }
1809
1810         nlmsg_end(msg, nlh);
1811         put_device(data.cdev);
1812         if (ibdev)
1813                 ib_device_put(ibdev);
1814         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1815
1816 out_data:
1817         put_device(data.cdev);
1818 out_nlmsg:
1819         nlmsg_free(msg);
1820 out_put:
1821         if (ibdev)
1822                 ib_device_put(ibdev);
1823         return err;
1824 }
1825
1826 static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1827                               struct netlink_ext_ack *extack)
1828 {
1829         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1830         struct sk_buff *msg;
1831         int err;
1832
1833         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1834                           nldev_policy, extack);
1835         if (err)
1836                 return err;
1837
1838         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1839         if (!msg)
1840                 return -ENOMEM;
1841
1842         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1843                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1844                                          RDMA_NLDEV_CMD_SYS_GET),
1845                         0, 0);
1846
1847         err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
1848                          (u8)ib_devices_shared_netns);
1849         if (err) {
1850                 nlmsg_free(msg);
1851                 return err;
1852         }
1853
1854         /*
1855          * Copy-on-fork is supported.
1856          * See commits:
1857          * 70e806e4e645 ("mm: Do early cow for pinned pages during fork() for ptes")
1858          * 4eae4efa2c29 ("hugetlb: do early cow when page pinned on src mm")
1859          * for more details. Don't backport this without them.
1860          *
1861          * Return value ignored on purpose, assume copy-on-fork is not
1862          * supported in case of failure.
1863          */
1864         nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK, 1);
1865
1866         nlmsg_end(msg, nlh);
1867         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1868 }
1869
1870 static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1871                                   struct netlink_ext_ack *extack)
1872 {
1873         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1874         u8 enable;
1875         int err;
1876
1877         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1878                           nldev_policy, extack);
1879         if (err || !tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])
1880                 return -EINVAL;
1881
1882         enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);
1883         /* Only 0 and 1 are supported */
1884         if (enable > 1)
1885                 return -EINVAL;
1886
1887         err = rdma_compatdev_set(enable);
1888         return err;
1889 }
1890
1891 static int nldev_stat_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1892                                struct netlink_ext_ack *extack)
1893 {
1894         u32 index, port, mode, mask = 0, qpn, cntn = 0;
1895         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1896         struct ib_device *device;
1897         struct sk_buff *msg;
1898         int ret;
1899
1900         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1901                           nldev_policy, extack);
1902         /* Currently only counter for QP is supported */
1903         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] ||
1904             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
1905             !tb[RDMA_NLDEV_ATTR_PORT_INDEX] || !tb[RDMA_NLDEV_ATTR_STAT_MODE])
1906                 return -EINVAL;
1907
1908         if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
1909                 return -EINVAL;
1910
1911         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1912         device = ib_device_get_by_index(sock_net(skb->sk), index);
1913         if (!device)
1914                 return -EINVAL;
1915
1916         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1917         if (!rdma_is_port_valid(device, port)) {
1918                 ret = -EINVAL;
1919                 goto err;
1920         }
1921
1922         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1923         if (!msg) {
1924                 ret = -ENOMEM;
1925                 goto err;
1926         }
1927         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1928                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1929                                          RDMA_NLDEV_CMD_STAT_SET),
1930                         0, 0);
1931
1932         mode = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_MODE]);
1933         if (mode == RDMA_COUNTER_MODE_AUTO) {
1934                 if (tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK])
1935                         mask = nla_get_u32(
1936                                 tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]);
1937                 ret = rdma_counter_set_auto_mode(device, port, mask, extack);
1938                 if (ret)
1939                         goto err_msg;
1940         } else {
1941                 if (!tb[RDMA_NLDEV_ATTR_RES_LQPN])
1942                         goto err_msg;
1943                 qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
1944                 if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) {
1945                         cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
1946                         ret = rdma_counter_bind_qpn(device, port, qpn, cntn);
1947                 } else {
1948                         ret = rdma_counter_bind_qpn_alloc(device, port,
1949                                                           qpn, &cntn);
1950                 }
1951                 if (ret)
1952                         goto err_msg;
1953
1954                 if (fill_nldev_handle(msg, device) ||
1955                     nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
1956                     nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
1957                     nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
1958                         ret = -EMSGSIZE;
1959                         goto err_fill;
1960                 }
1961         }
1962
1963         nlmsg_end(msg, nlh);
1964         ib_device_put(device);
1965         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1966
1967 err_fill:
1968         rdma_counter_unbind_qpn(device, port, qpn, cntn);
1969 err_msg:
1970         nlmsg_free(msg);
1971 err:
1972         ib_device_put(device);
1973         return ret;
1974 }
1975
1976 static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1977                                struct netlink_ext_ack *extack)
1978 {
1979         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1980         struct ib_device *device;
1981         struct sk_buff *msg;
1982         u32 index, port, qpn, cntn;
1983         int ret;
1984
1985         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1986                           nldev_policy, extack);
1987         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] ||
1988             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX] ||
1989             !tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID] ||
1990             !tb[RDMA_NLDEV_ATTR_RES_LQPN])
1991                 return -EINVAL;
1992
1993         if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
1994                 return -EINVAL;
1995
1996         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1997         device = ib_device_get_by_index(sock_net(skb->sk), index);
1998         if (!device)
1999                 return -EINVAL;
2000
2001         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2002         if (!rdma_is_port_valid(device, port)) {
2003                 ret = -EINVAL;
2004                 goto err;
2005         }
2006
2007         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2008         if (!msg) {
2009                 ret = -ENOMEM;
2010                 goto err;
2011         }
2012         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2013                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2014                                          RDMA_NLDEV_CMD_STAT_SET),
2015                         0, 0);
2016
2017         cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
2018         qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
2019         if (fill_nldev_handle(msg, device) ||
2020             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2021             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
2022             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
2023                 ret = -EMSGSIZE;
2024                 goto err_fill;
2025         }
2026
2027         ret = rdma_counter_unbind_qpn(device, port, qpn, cntn);
2028         if (ret)
2029                 goto err_fill;
2030
2031         nlmsg_end(msg, nlh);
2032         ib_device_put(device);
2033         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2034
2035 err_fill:
2036         nlmsg_free(msg);
2037 err:
2038         ib_device_put(device);
2039         return ret;
2040 }
2041
2042 static int stat_get_doit_default_counter(struct sk_buff *skb,
2043                                          struct nlmsghdr *nlh,
2044                                          struct netlink_ext_ack *extack,
2045                                          struct nlattr *tb[])
2046 {
2047         struct rdma_hw_stats *stats;
2048         struct nlattr *table_attr;
2049         struct ib_device *device;
2050         int ret, num_cnts, i;
2051         struct sk_buff *msg;
2052         u32 index, port;
2053         u64 v;
2054
2055         if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2056                 return -EINVAL;
2057
2058         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2059         device = ib_device_get_by_index(sock_net(skb->sk), index);
2060         if (!device)
2061                 return -EINVAL;
2062
2063         if (!device->ops.alloc_hw_stats || !device->ops.get_hw_stats) {
2064                 ret = -EINVAL;
2065                 goto err;
2066         }
2067
2068         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2069         if (!rdma_is_port_valid(device, port)) {
2070                 ret = -EINVAL;
2071                 goto err;
2072         }
2073
2074         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2075         if (!msg) {
2076                 ret = -ENOMEM;
2077                 goto err;
2078         }
2079
2080         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2081                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2082                                          RDMA_NLDEV_CMD_STAT_GET),
2083                         0, 0);
2084
2085         if (fill_nldev_handle(msg, device) ||
2086             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2087                 ret = -EMSGSIZE;
2088                 goto err_msg;
2089         }
2090
2091         stats = device->port_data ? device->port_data[port].hw_stats : NULL;
2092         if (stats == NULL) {
2093                 ret = -EINVAL;
2094                 goto err_msg;
2095         }
2096         mutex_lock(&stats->lock);
2097
2098         num_cnts = device->ops.get_hw_stats(device, stats, port, 0);
2099         if (num_cnts < 0) {
2100                 ret = -EINVAL;
2101                 goto err_stats;
2102         }
2103
2104         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
2105         if (!table_attr) {
2106                 ret = -EMSGSIZE;
2107                 goto err_stats;
2108         }
2109         for (i = 0; i < num_cnts; i++) {
2110                 v = stats->value[i] +
2111                         rdma_counter_get_hwstat_value(device, port, i);
2112                 if (rdma_nl_stat_hwcounter_entry(msg, stats->names[i], v)) {
2113                         ret = -EMSGSIZE;
2114                         goto err_table;
2115                 }
2116         }
2117         nla_nest_end(msg, table_attr);
2118
2119         mutex_unlock(&stats->lock);
2120         nlmsg_end(msg, nlh);
2121         ib_device_put(device);
2122         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2123
2124 err_table:
2125         nla_nest_cancel(msg, table_attr);
2126 err_stats:
2127         mutex_unlock(&stats->lock);
2128 err_msg:
2129         nlmsg_free(msg);
2130 err:
2131         ib_device_put(device);
2132         return ret;
2133 }
2134
2135 static int stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,
2136                             struct netlink_ext_ack *extack, struct nlattr *tb[])
2137
2138 {
2139         static enum rdma_nl_counter_mode mode;
2140         static enum rdma_nl_counter_mask mask;
2141         struct ib_device *device;
2142         struct sk_buff *msg;
2143         u32 index, port;
2144         int ret;
2145
2146         if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID])
2147                 return nldev_res_get_counter_doit(skb, nlh, extack);
2148
2149         if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] ||
2150             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2151                 return -EINVAL;
2152
2153         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2154         device = ib_device_get_by_index(sock_net(skb->sk), index);
2155         if (!device)
2156                 return -EINVAL;
2157
2158         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2159         if (!rdma_is_port_valid(device, port)) {
2160                 ret = -EINVAL;
2161                 goto err;
2162         }
2163
2164         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2165         if (!msg) {
2166                 ret = -ENOMEM;
2167                 goto err;
2168         }
2169
2170         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2171                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2172                                          RDMA_NLDEV_CMD_STAT_GET),
2173                         0, 0);
2174
2175         ret = rdma_counter_get_mode(device, port, &mode, &mask);
2176         if (ret)
2177                 goto err_msg;
2178
2179         if (fill_nldev_handle(msg, device) ||
2180             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2181             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, mode)) {
2182                 ret = -EMSGSIZE;
2183                 goto err_msg;
2184         }
2185
2186         if ((mode == RDMA_COUNTER_MODE_AUTO) &&
2187             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, mask)) {
2188                 ret = -EMSGSIZE;
2189                 goto err_msg;
2190         }
2191
2192         nlmsg_end(msg, nlh);
2193         ib_device_put(device);
2194         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2195
2196 err_msg:
2197         nlmsg_free(msg);
2198 err:
2199         ib_device_put(device);
2200         return ret;
2201 }
2202
2203 static int nldev_stat_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2204                                struct netlink_ext_ack *extack)
2205 {
2206         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2207         int ret;
2208
2209         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2210                           nldev_policy, extack);
2211         if (ret)
2212                 return -EINVAL;
2213
2214         if (!tb[RDMA_NLDEV_ATTR_STAT_RES])
2215                 return stat_get_doit_default_counter(skb, nlh, extack, tb);
2216
2217         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2218         case RDMA_NLDEV_ATTR_RES_QP:
2219                 ret = stat_get_doit_qp(skb, nlh, extack, tb);
2220                 break;
2221         case RDMA_NLDEV_ATTR_RES_MR:
2222                 ret = res_get_common_doit(skb, nlh, extack, RDMA_RESTRACK_MR,
2223                                           fill_stat_mr_entry);
2224                 break;
2225         default:
2226                 ret = -EINVAL;
2227                 break;
2228         }
2229
2230         return ret;
2231 }
2232
2233 static int nldev_stat_get_dumpit(struct sk_buff *skb,
2234                                  struct netlink_callback *cb)
2235 {
2236         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2237         int ret;
2238
2239         ret = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2240                           nldev_policy, NULL);
2241         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES])
2242                 return -EINVAL;
2243
2244         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2245         case RDMA_NLDEV_ATTR_RES_QP:
2246                 ret = nldev_res_get_counter_dumpit(skb, cb);
2247                 break;
2248         case RDMA_NLDEV_ATTR_RES_MR:
2249                 ret = res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR,
2250                                             fill_stat_mr_entry);
2251                 break;
2252         default:
2253                 ret = -EINVAL;
2254                 break;
2255         }
2256
2257         return ret;
2258 }
2259
2260 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
2261         [RDMA_NLDEV_CMD_GET] = {
2262                 .doit = nldev_get_doit,
2263                 .dump = nldev_get_dumpit,
2264         },
2265         [RDMA_NLDEV_CMD_GET_CHARDEV] = {
2266                 .doit = nldev_get_chardev,
2267         },
2268         [RDMA_NLDEV_CMD_SET] = {
2269                 .doit = nldev_set_doit,
2270                 .flags = RDMA_NL_ADMIN_PERM,
2271         },
2272         [RDMA_NLDEV_CMD_NEWLINK] = {
2273                 .doit = nldev_newlink,
2274                 .flags = RDMA_NL_ADMIN_PERM,
2275         },
2276         [RDMA_NLDEV_CMD_DELLINK] = {
2277                 .doit = nldev_dellink,
2278                 .flags = RDMA_NL_ADMIN_PERM,
2279         },
2280         [RDMA_NLDEV_CMD_PORT_GET] = {
2281                 .doit = nldev_port_get_doit,
2282                 .dump = nldev_port_get_dumpit,
2283         },
2284         [RDMA_NLDEV_CMD_RES_GET] = {
2285                 .doit = nldev_res_get_doit,
2286                 .dump = nldev_res_get_dumpit,
2287         },
2288         [RDMA_NLDEV_CMD_RES_QP_GET] = {
2289                 .doit = nldev_res_get_qp_doit,
2290                 .dump = nldev_res_get_qp_dumpit,
2291         },
2292         [RDMA_NLDEV_CMD_RES_CM_ID_GET] = {
2293                 .doit = nldev_res_get_cm_id_doit,
2294                 .dump = nldev_res_get_cm_id_dumpit,
2295         },
2296         [RDMA_NLDEV_CMD_RES_CQ_GET] = {
2297                 .doit = nldev_res_get_cq_doit,
2298                 .dump = nldev_res_get_cq_dumpit,
2299         },
2300         [RDMA_NLDEV_CMD_RES_MR_GET] = {
2301                 .doit = nldev_res_get_mr_doit,
2302                 .dump = nldev_res_get_mr_dumpit,
2303         },
2304         [RDMA_NLDEV_CMD_RES_PD_GET] = {
2305                 .doit = nldev_res_get_pd_doit,
2306                 .dump = nldev_res_get_pd_dumpit,
2307         },
2308         [RDMA_NLDEV_CMD_RES_CTX_GET] = {
2309                 .doit = nldev_res_get_ctx_doit,
2310                 .dump = nldev_res_get_ctx_dumpit,
2311         },
2312         [RDMA_NLDEV_CMD_RES_SRQ_GET] = {
2313                 .doit = nldev_res_get_srq_doit,
2314                 .dump = nldev_res_get_srq_dumpit,
2315         },
2316         [RDMA_NLDEV_CMD_SYS_GET] = {
2317                 .doit = nldev_sys_get_doit,
2318         },
2319         [RDMA_NLDEV_CMD_SYS_SET] = {
2320                 .doit = nldev_set_sys_set_doit,
2321         },
2322         [RDMA_NLDEV_CMD_STAT_SET] = {
2323                 .doit = nldev_stat_set_doit,
2324                 .flags = RDMA_NL_ADMIN_PERM,
2325         },
2326         [RDMA_NLDEV_CMD_STAT_GET] = {
2327                 .doit = nldev_stat_get_doit,
2328                 .dump = nldev_stat_get_dumpit,
2329         },
2330         [RDMA_NLDEV_CMD_STAT_DEL] = {
2331                 .doit = nldev_stat_del_doit,
2332                 .flags = RDMA_NL_ADMIN_PERM,
2333         },
2334         [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {
2335                 .doit = nldev_res_get_qp_raw_doit,
2336                 .dump = nldev_res_get_qp_raw_dumpit,
2337                 .flags = RDMA_NL_ADMIN_PERM,
2338         },
2339         [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {
2340                 .doit = nldev_res_get_cq_raw_doit,
2341                 .dump = nldev_res_get_cq_raw_dumpit,
2342                 .flags = RDMA_NL_ADMIN_PERM,
2343         },
2344         [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {
2345                 .doit = nldev_res_get_mr_raw_doit,
2346                 .dump = nldev_res_get_mr_raw_dumpit,
2347                 .flags = RDMA_NL_ADMIN_PERM,
2348         },
2349 };
2350
2351 void __init nldev_init(void)
2352 {
2353         rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table);
2354 }
2355
2356 void __exit nldev_exit(void)
2357 {
2358         rdma_nl_unregister(RDMA_NL_NLDEV);
2359 }
2360
2361 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_NLDEV, 5);