net/core/devlink.c

   1 /*
   2  * net/core/devlink.c - Network physical/parent device Netlink interface
   3  *
   4  * Heavily inspired by net/wireless/
   5  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   6  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17 #include <linux/slab.h>
  18 #include <linux/gfp.h>
  19 #include <linux/device.h>
  20 #include <linux/list.h>
  21 #include <linux/netdevice.h>
  22 #include <rdma/ib_verbs.h>
  23 #include <net/netlink.h>
  24 #include <net/genetlink.h>
  25 #include <net/rtnetlink.h>
  26 #include <net/net_namespace.h>
  27 #include <net/sock.h>
  28 #include <net/devlink.h>
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/devlink.h>
  31
  32 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
  33         {
  34                 .name = "destination mac",
  35                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
  36                 .bitwidth = 48,
  37         },
  38 };
  39
  40 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
  41         .name = "ethernet",
  42         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
  43         .fields = devlink_dpipe_fields_ethernet,
  44         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
  45         .global = true,
  46 };
  47 EXPORT_SYMBOL(devlink_dpipe_header_ethernet);
  48
  49 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
  50         {
  51                 .name = "destination ip",
  52                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
  53                 .bitwidth = 32,
  54         },
  55 };
  56
  57 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
  58         .name = "ipv4",
  59         .id = DEVLINK_DPIPE_HEADER_IPV4,
  60         .fields = devlink_dpipe_fields_ipv4,
  61         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
  62         .global = true,
  63 };
  64 EXPORT_SYMBOL(devlink_dpipe_header_ipv4);
  65
  66 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
  67
  68 static LIST_HEAD(devlink_list);
  69
  70 /* devlink_mutex
  71  *
  72  * An overall lock guarding every operation coming from userspace.
  73  * It also guards devlink devices list and it is taken when
  74  * driver registers/unregisters it.
  75  */
  76 static DEFINE_MUTEX(devlink_mutex);
  77
  78 /* devlink_port_mutex
  79  *
  80  * Shared lock to guard lists of ports in all devlink devices.
  81  */
  82 static DEFINE_MUTEX(devlink_port_mutex);
  83
  84 static struct net *devlink_net(const struct devlink *devlink)
  85 {
  86         return read_pnet(&devlink->_net);
  87 }
  88
  89 static void devlink_net_set(struct devlink *devlink, struct net *net)
  90 {
  91         write_pnet(&devlink->_net, net);
  92 }
  93
  94 static struct devlink *devlink_get_from_attrs(struct net *net,
  95                                               struct nlattr **attrs)
  96 {
  97         struct devlink *devlink;
  98         char *busname;
  99         char *devname;
 100
 101         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 102                 return ERR_PTR(-EINVAL);
 103
 104         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 105         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 106
 107         list_for_each_entry(devlink, &devlink_list, list) {
 108                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 109                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 110                     net_eq(devlink_net(devlink), net))
 111                         return devlink;
 112         }
 113
 114         return ERR_PTR(-ENODEV);
 115 }
 116
 117 static struct devlink *devlink_get_from_info(struct genl_info *info)
 118 {
 119         return devlink_get_from_attrs(genl_info_net(info), info->attrs);
 120 }
 121
 122 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 123                                                       int port_index)
 124 {
 125         struct devlink_port *devlink_port;
 126
 127         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 128                 if (devlink_port->index == port_index)
 129                         return devlink_port;
 130         }
 131         return NULL;
 132 }
 133
 134 static bool devlink_port_index_exists(struct devlink *devlink, int port_index)
 135 {
 136         return devlink_port_get_by_index(devlink, port_index);
 137 }
 138
 139 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 140                                                         struct nlattr **attrs)
 141 {
 142         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 143                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 144                 struct devlink_port *devlink_port;
 145
 146                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 147                 if (!devlink_port)
 148                         return ERR_PTR(-ENODEV);
 149                 return devlink_port;
 150         }
 151         return ERR_PTR(-EINVAL);
 152 }
 153
 154 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 155                                                        struct genl_info *info)
 156 {
 157         return devlink_port_get_from_attrs(devlink, info->attrs);
 158 }
 159
 160 struct devlink_sb {
 161         struct list_head list;
 162         unsigned int index;
 163         u32 size;
 164         u16 ingress_pools_count;
 165         u16 egress_pools_count;
 166         u16 ingress_tc_count;
 167         u16 egress_tc_count;
 168 };
 169
 170 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 171 {
 172         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 173 }
 174
 175 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 176                                                   unsigned int sb_index)
 177 {
 178         struct devlink_sb *devlink_sb;
 179
 180         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 181                 if (devlink_sb->index == sb_index)
 182                         return devlink_sb;
 183         }
 184         return NULL;
 185 }
 186
 187 static bool devlink_sb_index_exists(struct devlink *devlink,
 188                                     unsigned int sb_index)
 189 {
 190         return devlink_sb_get_by_index(devlink, sb_index);
 191 }
 192
 193 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 194                                                     struct nlattr **attrs)
 195 {
 196         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 197                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 198                 struct devlink_sb *devlink_sb;
 199
 200                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 201                 if (!devlink_sb)
 202                         return ERR_PTR(-ENODEV);
 203                 return devlink_sb;
 204         }
 205         return ERR_PTR(-EINVAL);
 206 }
 207
 208 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 209                                                    struct genl_info *info)
 210 {
 211         return devlink_sb_get_from_attrs(devlink, info->attrs);
 212 }
 213
 214 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 215                                                 struct nlattr **attrs,
 216                                                 u16 *p_pool_index)
 217 {
 218         u16 val;
 219
 220         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 221                 return -EINVAL;
 222
 223         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 224         if (val >= devlink_sb_pool_count(devlink_sb))
 225                 return -EINVAL;
 226         *p_pool_index = val;
 227         return 0;
 228 }
 229
 230 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 231                                                struct genl_info *info,
 232                                                u16 *p_pool_index)
 233 {
 234         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 235                                                     p_pool_index);
 236 }
 237
 238 static int
 239 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 240                                     enum devlink_sb_pool_type *p_pool_type)
 241 {
 242         u8 val;
 243
 244         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 245                 return -EINVAL;
 246
 247         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 248         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 249             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 250                 return -EINVAL;
 251         *p_pool_type = val;
 252         return 0;
 253 }
 254
 255 static int
 256 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 257                                    enum devlink_sb_pool_type *p_pool_type)
 258 {
 259         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 260 }
 261
 262 static int
 263 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 264                                   enum devlink_sb_threshold_type *p_th_type)
 265 {
 266         u8 val;
 267
 268         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 269                 return -EINVAL;
 270
 271         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 272         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 273             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 274                 return -EINVAL;
 275         *p_th_type = val;
 276         return 0;
 277 }
 278
 279 static int
 280 devlink_sb_th_type_get_from_info(struct genl_info *info,
 281                                  enum devlink_sb_threshold_type *p_th_type)
 282 {
 283         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 284 }
 285
 286 static int
 287 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 288                                    struct nlattr **attrs,
 289                                    enum devlink_sb_pool_type pool_type,
 290                                    u16 *p_tc_index)
 291 {
 292         u16 val;
 293
 294         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 295                 return -EINVAL;
 296
 297         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 298         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 299             val >= devlink_sb->ingress_tc_count)
 300                 return -EINVAL;
 301         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 302             val >= devlink_sb->egress_tc_count)
 303                 return -EINVAL;
 304         *p_tc_index = val;
 305         return 0;
 306 }
 307
 308 static int
 309 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 310                                   struct genl_info *info,
 311                                   enum devlink_sb_pool_type pool_type,
 312                                   u16 *p_tc_index)
 313 {
 314         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 315                                                   pool_type, p_tc_index);
 316 }
 317
 318 #define DEVLINK_NL_FLAG_NEED_DEVLINK    BIT(0)
 319 #define DEVLINK_NL_FLAG_NEED_PORT       BIT(1)
 320 #define DEVLINK_NL_FLAG_NEED_SB         BIT(2)
 321 #define DEVLINK_NL_FLAG_LOCK_PORTS      BIT(3)
 322         /* port is not needed but we need to ensure they don't
 323          * change in the middle of command
 324          */
 325
 326 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 327                                struct sk_buff *skb, struct genl_info *info)
 328 {
 329         struct devlink *devlink;
 330
 331         mutex_lock(&devlink_mutex);
 332         devlink = devlink_get_from_info(info);
 333         if (IS_ERR(devlink)) {
 334                 mutex_unlock(&devlink_mutex);
 335                 return PTR_ERR(devlink);
 336         }
 337         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK) {
 338                 info->user_ptr[0] = devlink;
 339         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 340                 struct devlink_port *devlink_port;
 341
 342                 mutex_lock(&devlink_port_mutex);
 343                 devlink_port = devlink_port_get_from_info(devlink, info);
 344                 if (IS_ERR(devlink_port)) {
 345                         mutex_unlock(&devlink_port_mutex);
 346                         mutex_unlock(&devlink_mutex);
 347                         return PTR_ERR(devlink_port);
 348                 }
 349                 info->user_ptr[0] = devlink_port;
 350         }
 351         if (ops->internal_flags & DEVLINK_NL_FLAG_LOCK_PORTS) {
 352                 mutex_lock(&devlink_port_mutex);
 353         }
 354         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_SB) {
 355                 struct devlink_sb *devlink_sb;
 356
 357                 devlink_sb = devlink_sb_get_from_info(devlink, info);
 358                 if (IS_ERR(devlink_sb)) {
 359                         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT)
 360                                 mutex_unlock(&devlink_port_mutex);
 361                         mutex_unlock(&devlink_mutex);
 362                         return PTR_ERR(devlink_sb);
 363                 }
 364                 info->user_ptr[1] = devlink_sb;
 365         }
 366         return 0;
 367 }
 368
 369 static void devlink_nl_post_doit(const struct genl_ops *ops,
 370                                  struct sk_buff *skb, struct genl_info *info)
 371 {
 372         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT ||
 373             ops->internal_flags & DEVLINK_NL_FLAG_LOCK_PORTS)
 374                 mutex_unlock(&devlink_port_mutex);
 375         mutex_unlock(&devlink_mutex);
 376 }
 377
 378 static struct genl_family devlink_nl_family;
 379
 380 enum devlink_multicast_groups {
 381         DEVLINK_MCGRP_CONFIG,
 382 };
 383
 384 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 385         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 386 };
 387
 388 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 389 {
 390         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 391                 return -EMSGSIZE;
 392         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 393                 return -EMSGSIZE;
 394         return 0;
 395 }
 396
 397 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 398                            enum devlink_command cmd, u32 portid,
 399                            u32 seq, int flags)
 400 {
 401         void *hdr;
 402
 403         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 404         if (!hdr)
 405                 return -EMSGSIZE;
 406
 407         if (devlink_nl_put_handle(msg, devlink))
 408                 goto nla_put_failure;
 409
 410         genlmsg_end(msg, hdr);
 411         return 0;
 412
 413 nla_put_failure:
 414         genlmsg_cancel(msg, hdr);
 415         return -EMSGSIZE;
 416 }
 417
 418 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 419 {
 420         struct sk_buff *msg;
 421         int err;
 422
 423         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 424
 425         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 426         if (!msg)
 427                 return;
 428
 429         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 430         if (err) {
 431                 nlmsg_free(msg);
 432                 return;
 433         }
 434
 435         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 436                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 437 }
 438
 439 static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
 440                                 struct devlink_port *devlink_port,
 441                                 enum devlink_command cmd, u32 portid,
 442                                 u32 seq, int flags)
 443 {
 444         void *hdr;
 445
 446         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 447         if (!hdr)
 448                 return -EMSGSIZE;
 449
 450         if (devlink_nl_put_handle(msg, devlink))
 451                 goto nla_put_failure;
 452         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
 453                 goto nla_put_failure;
 454         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
 455                 goto nla_put_failure;
 456         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
 457             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
 458                         devlink_port->desired_type))
 459                 goto nla_put_failure;
 460         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
 461                 struct net_device *netdev = devlink_port->type_dev;
 462
 463                 if (netdev &&
 464                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
 465                                  netdev->ifindex) ||
 466                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
 467                                     netdev->name)))
 468                         goto nla_put_failure;
 469         }
 470         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
 471                 struct ib_device *ibdev = devlink_port->type_dev;
 472
 473                 if (ibdev &&
 474                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
 475                                    ibdev->name))
 476                         goto nla_put_failure;
 477         }
 478         if (devlink_port->split &&
 479             nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
 480                         devlink_port->split_group))
 481                 goto nla_put_failure;
 482
 483         genlmsg_end(msg, hdr);
 484         return 0;
 485
 486 nla_put_failure:
 487         genlmsg_cancel(msg, hdr);
 488         return -EMSGSIZE;
 489 }
 490
 491 static void devlink_port_notify(struct devlink_port *devlink_port,
 492                                 enum devlink_command cmd)
 493 {
 494         struct devlink *devlink = devlink_port->devlink;
 495         struct sk_buff *msg;
 496         int err;
 497
 498         if (!devlink_port->registered)
 499                 return;
 500
 501         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
 502
 503         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 504         if (!msg)
 505                 return;
 506
 507         err = devlink_nl_port_fill(msg, devlink, devlink_port, cmd, 0, 0, 0);
 508         if (err) {
 509                 nlmsg_free(msg);
 510                 return;
 511         }
 512
 513         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 514                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 515 }
 516
 517 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
 518 {
 519         struct devlink *devlink = info->user_ptr[0];
 520         struct sk_buff *msg;
 521         int err;
 522
 523         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 524         if (!msg)
 525                 return -ENOMEM;
 526
 527         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 528                               info->snd_portid, info->snd_seq, 0);
 529         if (err) {
 530                 nlmsg_free(msg);
 531                 return err;
 532         }
 533
 534         return genlmsg_reply(msg, info);
 535 }
 536
 537 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
 538                                      struct netlink_callback *cb)
 539 {
 540         struct devlink *devlink;
 541         int start = cb->args[0];
 542         int idx = 0;
 543         int err;
 544
 545         mutex_lock(&devlink_mutex);
 546         list_for_each_entry(devlink, &devlink_list, list) {
 547                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 548                         continue;
 549                 if (idx < start) {
 550                         idx++;
 551                         continue;
 552                 }
 553                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 554                                       NETLINK_CB(cb->skb).portid,
 555                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
 556                 if (err)
 557                         goto out;
 558                 idx++;
 559         }
 560 out:
 561         mutex_unlock(&devlink_mutex);
 562
 563         cb->args[0] = idx;
 564         return msg->len;
 565 }
 566
 567 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
 568                                         struct genl_info *info)
 569 {
 570         struct devlink_port *devlink_port = info->user_ptr[0];
 571         struct devlink *devlink = devlink_port->devlink;
 572         struct sk_buff *msg;
 573         int err;
 574
 575         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 576         if (!msg)
 577                 return -ENOMEM;
 578
 579         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 580                                    DEVLINK_CMD_PORT_NEW,
 581                                    info->snd_portid, info->snd_seq, 0);
 582         if (err) {
 583                 nlmsg_free(msg);
 584                 return err;
 585         }
 586
 587         return genlmsg_reply(msg, info);
 588 }
 589
 590 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
 591                                           struct netlink_callback *cb)
 592 {
 593         struct devlink *devlink;
 594         struct devlink_port *devlink_port;
 595         int start = cb->args[0];
 596         int idx = 0;
 597         int err;
 598
 599         mutex_lock(&devlink_mutex);
 600         mutex_lock(&devlink_port_mutex);
 601         list_for_each_entry(devlink, &devlink_list, list) {
 602                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 603                         continue;
 604                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
 605                         if (idx < start) {
 606                                 idx++;
 607                                 continue;
 608                         }
 609                         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 610                                                    DEVLINK_CMD_NEW,
 611                                                    NETLINK_CB(cb->skb).portid,
 612                                                    cb->nlh->nlmsg_seq,
 613                                                    NLM_F_MULTI);
 614                         if (err)
 615                                 goto out;
 616                         idx++;
 617                 }
 618         }
 619 out:
 620         mutex_unlock(&devlink_port_mutex);
 621         mutex_unlock(&devlink_mutex);
 622
 623         cb->args[0] = idx;
 624         return msg->len;
 625 }
 626
 627 static int devlink_port_type_set(struct devlink *devlink,
 628                                  struct devlink_port *devlink_port,
 629                                  enum devlink_port_type port_type)
 630
 631 {
 632         int err;
 633
 634         if (devlink->ops && devlink->ops->port_type_set) {
 635                 if (port_type == DEVLINK_PORT_TYPE_NOTSET)
 636                         return -EINVAL;
 637                 if (port_type == devlink_port->type)
 638                         return 0;
 639                 err = devlink->ops->port_type_set(devlink_port, port_type);
 640                 if (err)
 641                         return err;
 642                 devlink_port->desired_type = port_type;
 643                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
 644                 return 0;
 645         }
 646         return -EOPNOTSUPP;
 647 }
 648
 649 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
 650                                         struct genl_info *info)
 651 {
 652         struct devlink_port *devlink_port = info->user_ptr[0];
 653         struct devlink *devlink = devlink_port->devlink;
 654         int err;
 655
 656         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
 657                 enum devlink_port_type port_type;
 658
 659                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
 660                 err = devlink_port_type_set(devlink, devlink_port, port_type);
 661                 if (err)
 662                         return err;
 663         }
 664         return 0;
 665 }
 666
 667 static int devlink_port_split(struct devlink *devlink,
 668                               u32 port_index, u32 count)
 669
 670 {
 671         if (devlink->ops && devlink->ops->port_split)
 672                 return devlink->ops->port_split(devlink, port_index, count);
 673         return -EOPNOTSUPP;
 674 }
 675
 676 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
 677                                           struct genl_info *info)
 678 {
 679         struct devlink *devlink = info->user_ptr[0];
 680         u32 port_index;
 681         u32 count;
 682
 683         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX] ||
 684             !info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
 685                 return -EINVAL;
 686
 687         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
 688         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
 689         return devlink_port_split(devlink, port_index, count);
 690 }
 691
 692 static int devlink_port_unsplit(struct devlink *devlink, u32 port_index)
 693
 694 {
 695         if (devlink->ops && devlink->ops->port_unsplit)
 696                 return devlink->ops->port_unsplit(devlink, port_index);
 697         return -EOPNOTSUPP;
 698 }
 699
 700 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
 701                                             struct genl_info *info)
 702 {
 703         struct devlink *devlink = info->user_ptr[0];
 704         u32 port_index;
 705
 706         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX])
 707                 return -EINVAL;
 708
 709         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
 710         return devlink_port_unsplit(devlink, port_index);
 711 }
 712
 713 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
 714                               struct devlink_sb *devlink_sb,
 715                               enum devlink_command cmd, u32 portid,
 716                               u32 seq, int flags)
 717 {
 718         void *hdr;
 719
 720         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 721         if (!hdr)
 722                 return -EMSGSIZE;
 723
 724         if (devlink_nl_put_handle(msg, devlink))
 725                 goto nla_put_failure;
 726         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
 727                 goto nla_put_failure;
 728         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
 729                 goto nla_put_failure;
 730         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
 731                         devlink_sb->ingress_pools_count))
 732                 goto nla_put_failure;
 733         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
 734                         devlink_sb->egress_pools_count))
 735                 goto nla_put_failure;
 736         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
 737                         devlink_sb->ingress_tc_count))
 738                 goto nla_put_failure;
 739         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
 740                         devlink_sb->egress_tc_count))
 741                 goto nla_put_failure;
 742
 743         genlmsg_end(msg, hdr);
 744         return 0;
 745
 746 nla_put_failure:
 747         genlmsg_cancel(msg, hdr);
 748         return -EMSGSIZE;
 749 }
 750
 751 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
 752                                       struct genl_info *info)
 753 {
 754         struct devlink *devlink = info->user_ptr[0];
 755         struct devlink_sb *devlink_sb = info->user_ptr[1];
 756         struct sk_buff *msg;
 757         int err;
 758
 759         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 760         if (!msg)
 761                 return -ENOMEM;
 762
 763         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
 764                                  DEVLINK_CMD_SB_NEW,
 765                                  info->snd_portid, info->snd_seq, 0);
 766         if (err) {
 767                 nlmsg_free(msg);
 768                 return err;
 769         }
 770
 771         return genlmsg_reply(msg, info);
 772 }
 773
 774 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
 775                                         struct netlink_callback *cb)
 776 {
 777         struct devlink *devlink;
 778         struct devlink_sb *devlink_sb;
 779         int start = cb->args[0];
 780         int idx = 0;
 781         int err;
 782
 783         mutex_lock(&devlink_mutex);
 784         list_for_each_entry(devlink, &devlink_list, list) {
 785                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 786                         continue;
 787                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 788                         if (idx < start) {
 789                                 idx++;
 790                                 continue;
 791                         }
 792                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
 793                                                  DEVLINK_CMD_SB_NEW,
 794                                                  NETLINK_CB(cb->skb).portid,
 795                                                  cb->nlh->nlmsg_seq,
 796                                                  NLM_F_MULTI);
 797                         if (err)
 798                                 goto out;
 799                         idx++;
 800                 }
 801         }
 802 out:
 803         mutex_unlock(&devlink_mutex);
 804
 805         cb->args[0] = idx;
 806         return msg->len;
 807 }
 808
 809 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
 810                                    struct devlink_sb *devlink_sb,
 811                                    u16 pool_index, enum devlink_command cmd,
 812                                    u32 portid, u32 seq, int flags)
 813 {
 814         struct devlink_sb_pool_info pool_info;
 815         void *hdr;
 816         int err;
 817
 818         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
 819                                         pool_index, &pool_info);
 820         if (err)
 821                 return err;
 822
 823         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 824         if (!hdr)
 825                 return -EMSGSIZE;
 826
 827         if (devlink_nl_put_handle(msg, devlink))
 828                 goto nla_put_failure;
 829         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
 830                 goto nla_put_failure;
 831         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
 832                 goto nla_put_failure;
 833         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
 834                 goto nla_put_failure;
 835         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
 836                 goto nla_put_failure;
 837         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
 838                        pool_info.threshold_type))
 839                 goto nla_put_failure;
 840
 841         genlmsg_end(msg, hdr);
 842         return 0;
 843
 844 nla_put_failure:
 845         genlmsg_cancel(msg, hdr);
 846         return -EMSGSIZE;
 847 }
 848
 849 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
 850                                            struct genl_info *info)
 851 {
 852         struct devlink *devlink = info->user_ptr[0];
 853         struct devlink_sb *devlink_sb = info->user_ptr[1];
 854         struct sk_buff *msg;
 855         u16 pool_index;
 856         int err;
 857
 858         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
 859                                                   &pool_index);
 860         if (err)
 861                 return err;
 862
 863         if (!devlink->ops || !devlink->ops->sb_pool_get)
 864                 return -EOPNOTSUPP;
 865
 866         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 867         if (!msg)
 868                 return -ENOMEM;
 869
 870         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
 871                                       DEVLINK_CMD_SB_POOL_NEW,
 872                                       info->snd_portid, info->snd_seq, 0);
 873         if (err) {
 874                 nlmsg_free(msg);
 875                 return err;
 876         }
 877
 878         return genlmsg_reply(msg, info);
 879 }
 880
 881 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
 882                                 struct devlink *devlink,
 883                                 struct devlink_sb *devlink_sb,
 884                                 u32 portid, u32 seq)
 885 {
 886         u16 pool_count = devlink_sb_pool_count(devlink_sb);
 887         u16 pool_index;
 888         int err;
 889
 890         for (pool_index = 0; pool_index < pool_count; pool_index++) {
 891                 if (*p_idx < start) {
 892                         (*p_idx)++;
 893                         continue;
 894                 }
 895                 err = devlink_nl_sb_pool_fill(msg, devlink,
 896                                               devlink_sb,
 897                                               pool_index,
 898                                               DEVLINK_CMD_SB_POOL_NEW,
 899                                               portid, seq, NLM_F_MULTI);
 900                 if (err)
 901                         return err;
 902                 (*p_idx)++;
 903         }
 904         return 0;
 905 }
 906
 907 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
 908                                              struct netlink_callback *cb)
 909 {
 910         struct devlink *devlink;
 911         struct devlink_sb *devlink_sb;
 912         int start = cb->args[0];
 913         int idx = 0;
 914         int err;
 915
 916         mutex_lock(&devlink_mutex);
 917         list_for_each_entry(devlink, &devlink_list, list) {
 918                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
 919                     !devlink->ops || !devlink->ops->sb_pool_get)
 920                         continue;
 921                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 922                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
 923                                                    devlink_sb,
 924                                                    NETLINK_CB(cb->skb).portid,
 925                                                    cb->nlh->nlmsg_seq);
 926                         if (err && err != -EOPNOTSUPP)
 927                                 goto out;
 928                 }
 929         }
 930 out:
 931         mutex_unlock(&devlink_mutex);
 932
 933         cb->args[0] = idx;
 934         return msg->len;
 935 }
 936
 937 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
 938                                u16 pool_index, u32 size,
 939                                enum devlink_sb_threshold_type threshold_type)
 940
 941 {
 942         const struct devlink_ops *ops = devlink->ops;
 943
 944         if (ops && ops->sb_pool_set)
 945                 return ops->sb_pool_set(devlink, sb_index, pool_index,
 946                                         size, threshold_type);
 947         return -EOPNOTSUPP;
 948 }
 949
 950 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
 951                                            struct genl_info *info)
 952 {
 953         struct devlink *devlink = info->user_ptr[0];
 954         struct devlink_sb *devlink_sb = info->user_ptr[1];
 955         enum devlink_sb_threshold_type threshold_type;
 956         u16 pool_index;
 957         u32 size;
 958         int err;
 959
 960         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
 961                                                   &pool_index);
 962         if (err)
 963                 return err;
 964
 965         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
 966         if (err)
 967                 return err;
 968
 969         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
 970                 return -EINVAL;
 971
 972         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
 973         return devlink_sb_pool_set(devlink, devlink_sb->index,
 974                                    pool_index, size, threshold_type);
 975 }
 976
 977 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
 978                                         struct devlink *devlink,
 979                                         struct devlink_port *devlink_port,
 980                                         struct devlink_sb *devlink_sb,
 981                                         u16 pool_index,
 982                                         enum devlink_command cmd,
 983                                         u32 portid, u32 seq, int flags)
 984 {
 985         const struct devlink_ops *ops = devlink->ops;
 986         u32 threshold;
 987         void *hdr;
 988         int err;
 989
 990         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
 991                                     pool_index, &threshold);
 992         if (err)
 993                 return err;
 994
 995         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 996         if (!hdr)
 997                 return -EMSGSIZE;
 998
 999         if (devlink_nl_put_handle(msg, devlink))
1000                 goto nla_put_failure;
1001         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1002                 goto nla_put_failure;
1003         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1004                 goto nla_put_failure;
1005         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1006                 goto nla_put_failure;
1007         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1008                 goto nla_put_failure;
1009
1010         if (ops->sb_occ_port_pool_get) {
1011                 u32 cur;
1012                 u32 max;
1013
1014                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
1015                                                 pool_index, &cur, &max);
1016                 if (err && err != -EOPNOTSUPP)
1017                         return err;
1018                 if (!err) {
1019                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1020                                 goto nla_put_failure;
1021                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1022                                 goto nla_put_failure;
1023                 }
1024         }
1025
1026         genlmsg_end(msg, hdr);
1027         return 0;
1028
1029 nla_put_failure:
1030         genlmsg_cancel(msg, hdr);
1031         return -EMSGSIZE;
1032 }
1033
1034 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
1035                                                 struct genl_info *info)
1036 {
1037         struct devlink_port *devlink_port = info->user_ptr[0];
1038         struct devlink *devlink = devlink_port->devlink;
1039         struct devlink_sb *devlink_sb = info->user_ptr[1];
1040         struct sk_buff *msg;
1041         u16 pool_index;
1042         int err;
1043
1044         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1045                                                   &pool_index);
1046         if (err)
1047                 return err;
1048
1049         if (!devlink->ops || !devlink->ops->sb_port_pool_get)
1050                 return -EOPNOTSUPP;
1051
1052         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1053         if (!msg)
1054                 return -ENOMEM;
1055
1056         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
1057                                            devlink_sb, pool_index,
1058                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1059                                            info->snd_portid, info->snd_seq, 0);
1060         if (err) {
1061                 nlmsg_free(msg);
1062                 return err;
1063         }
1064
1065         return genlmsg_reply(msg, info);
1066 }
1067
1068 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
1069                                      struct devlink *devlink,
1070                                      struct devlink_sb *devlink_sb,
1071                                      u32 portid, u32 seq)
1072 {
1073         struct devlink_port *devlink_port;
1074         u16 pool_count = devlink_sb_pool_count(devlink_sb);
1075         u16 pool_index;
1076         int err;
1077
1078         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1079                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
1080                         if (*p_idx < start) {
1081                                 (*p_idx)++;
1082                                 continue;
1083                         }
1084                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
1085                                                            devlink_port,
1086                                                            devlink_sb,
1087                                                            pool_index,
1088                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1089                                                            portid, seq,
1090                                                            NLM_F_MULTI);
1091                         if (err)
1092                                 return err;
1093                         (*p_idx)++;
1094                 }
1095         }
1096         return 0;
1097 }
1098
1099 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
1100                                                   struct netlink_callback *cb)
1101 {
1102         struct devlink *devlink;
1103         struct devlink_sb *devlink_sb;
1104         int start = cb->args[0];
1105         int idx = 0;
1106         int err;
1107
1108         mutex_lock(&devlink_mutex);
1109         mutex_lock(&devlink_port_mutex);
1110         list_for_each_entry(devlink, &devlink_list, list) {
1111                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1112                     !devlink->ops || !devlink->ops->sb_port_pool_get)
1113                         continue;
1114                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1115                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
1116                                                         devlink, devlink_sb,
1117                                                         NETLINK_CB(cb->skb).portid,
1118                                                         cb->nlh->nlmsg_seq);
1119                         if (err && err != -EOPNOTSUPP)
1120                                 goto out;
1121                 }
1122         }
1123 out:
1124         mutex_unlock(&devlink_port_mutex);
1125         mutex_unlock(&devlink_mutex);
1126
1127         cb->args[0] = idx;
1128         return msg->len;
1129 }
1130
1131 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
1132                                     unsigned int sb_index, u16 pool_index,
1133                                     u32 threshold)
1134
1135 {
1136         const struct devlink_ops *ops = devlink_port->devlink->ops;
1137
1138         if (ops && ops->sb_port_pool_set)
1139                 return ops->sb_port_pool_set(devlink_port, sb_index,
1140                                              pool_index, threshold);
1141         return -EOPNOTSUPP;
1142 }
1143
1144 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
1145                                                 struct genl_info *info)
1146 {
1147         struct devlink_port *devlink_port = info->user_ptr[0];
1148         struct devlink_sb *devlink_sb = info->user_ptr[1];
1149         u16 pool_index;
1150         u32 threshold;
1151         int err;
1152
1153         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1154                                                   &pool_index);
1155         if (err)
1156                 return err;
1157
1158         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1159                 return -EINVAL;
1160
1161         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1162         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
1163                                         pool_index, threshold);
1164 }
1165
1166 static int
1167 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
1168                                 struct devlink_port *devlink_port,
1169                                 struct devlink_sb *devlink_sb, u16 tc_index,
1170                                 enum devlink_sb_pool_type pool_type,
1171                                 enum devlink_command cmd,
1172                                 u32 portid, u32 seq, int flags)
1173 {
1174         const struct devlink_ops *ops = devlink->ops;
1175         u16 pool_index;
1176         u32 threshold;
1177         void *hdr;
1178         int err;
1179
1180         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
1181                                        tc_index, pool_type,
1182                                        &pool_index, &threshold);
1183         if (err)
1184                 return err;
1185
1186         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1187         if (!hdr)
1188                 return -EMSGSIZE;
1189
1190         if (devlink_nl_put_handle(msg, devlink))
1191                 goto nla_put_failure;
1192         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1193                 goto nla_put_failure;
1194         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1195                 goto nla_put_failure;
1196         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
1197                 goto nla_put_failure;
1198         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
1199                 goto nla_put_failure;
1200         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1201                 goto nla_put_failure;
1202         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1203                 goto nla_put_failure;
1204
1205         if (ops->sb_occ_tc_port_bind_get) {
1206                 u32 cur;
1207                 u32 max;
1208
1209                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
1210                                                    devlink_sb->index,
1211                                                    tc_index, pool_type,
1212                                                    &cur, &max);
1213                 if (err && err != -EOPNOTSUPP)
1214                         return err;
1215                 if (!err) {
1216                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1217                                 goto nla_put_failure;
1218                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1219                                 goto nla_put_failure;
1220                 }
1221         }
1222
1223         genlmsg_end(msg, hdr);
1224         return 0;
1225
1226 nla_put_failure:
1227         genlmsg_cancel(msg, hdr);
1228         return -EMSGSIZE;
1229 }
1230
1231 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
1232                                                    struct genl_info *info)
1233 {
1234         struct devlink_port *devlink_port = info->user_ptr[0];
1235         struct devlink *devlink = devlink_port->devlink;
1236         struct devlink_sb *devlink_sb = info->user_ptr[1];
1237         struct sk_buff *msg;
1238         enum devlink_sb_pool_type pool_type;
1239         u16 tc_index;
1240         int err;
1241
1242         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1243         if (err)
1244                 return err;
1245
1246         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1247                                                 pool_type, &tc_index);
1248         if (err)
1249                 return err;
1250
1251         if (!devlink->ops || !devlink->ops->sb_tc_pool_bind_get)
1252                 return -EOPNOTSUPP;
1253
1254         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1255         if (!msg)
1256                 return -ENOMEM;
1257
1258         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
1259                                               devlink_sb, tc_index, pool_type,
1260                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1261                                               info->snd_portid,
1262                                               info->snd_seq, 0);
1263         if (err) {
1264                 nlmsg_free(msg);
1265                 return err;
1266         }
1267
1268         return genlmsg_reply(msg, info);
1269 }
1270
1271 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1272                                         int start, int *p_idx,
1273                                         struct devlink *devlink,
1274                                         struct devlink_sb *devlink_sb,
1275                                         u32 portid, u32 seq)
1276 {
1277         struct devlink_port *devlink_port;
1278         u16 tc_index;
1279         int err;
1280
1281         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1282                 for (tc_index = 0;
1283                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
1284                         if (*p_idx < start) {
1285                                 (*p_idx)++;
1286                                 continue;
1287                         }
1288                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1289                                                               devlink_port,
1290                                                               devlink_sb,
1291                                                               tc_index,
1292                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
1293                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1294                                                               portid, seq,
1295                                                               NLM_F_MULTI);
1296                         if (err)
1297                                 return err;
1298                         (*p_idx)++;
1299                 }
1300                 for (tc_index = 0;
1301                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
1302                         if (*p_idx < start) {
1303                                 (*p_idx)++;
1304                                 continue;
1305                         }
1306                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1307                                                               devlink_port,
1308                                                               devlink_sb,
1309                                                               tc_index,
1310                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
1311                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1312                                                               portid, seq,
1313                                                               NLM_F_MULTI);
1314                         if (err)
1315                                 return err;
1316                         (*p_idx)++;
1317                 }
1318         }
1319         return 0;
1320 }
1321
1322 static int
1323 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1324                                           struct netlink_callback *cb)
1325 {
1326         struct devlink *devlink;
1327         struct devlink_sb *devlink_sb;
1328         int start = cb->args[0];
1329         int idx = 0;
1330         int err;
1331
1332         mutex_lock(&devlink_mutex);
1333         mutex_lock(&devlink_port_mutex);
1334         list_for_each_entry(devlink, &devlink_list, list) {
1335                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1336                     !devlink->ops || !devlink->ops->sb_tc_pool_bind_get)
1337                         continue;
1338                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1339                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
1340                                                            devlink,
1341                                                            devlink_sb,
1342                                                            NETLINK_CB(cb->skb).portid,
1343                                                            cb->nlh->nlmsg_seq);
1344                         if (err && err != -EOPNOTSUPP)
1345                                 goto out;
1346                 }
1347         }
1348 out:
1349         mutex_unlock(&devlink_port_mutex);
1350         mutex_unlock(&devlink_mutex);
1351
1352         cb->args[0] = idx;
1353         return msg->len;
1354 }
1355
1356 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1357                                        unsigned int sb_index, u16 tc_index,
1358                                        enum devlink_sb_pool_type pool_type,
1359                                        u16 pool_index, u32 threshold)
1360
1361 {
1362         const struct devlink_ops *ops = devlink_port->devlink->ops;
1363
1364         if (ops && ops->sb_tc_pool_bind_set)
1365                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
1366                                                 tc_index, pool_type,
1367                                                 pool_index, threshold);
1368         return -EOPNOTSUPP;
1369 }
1370
1371 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
1372                                                    struct genl_info *info)
1373 {
1374         struct devlink_port *devlink_port = info->user_ptr[0];
1375         struct devlink_sb *devlink_sb = info->user_ptr[1];
1376         enum devlink_sb_pool_type pool_type;
1377         u16 tc_index;
1378         u16 pool_index;
1379         u32 threshold;
1380         int err;
1381
1382         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1383         if (err)
1384                 return err;
1385
1386         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1387                                                 pool_type, &tc_index);
1388         if (err)
1389                 return err;
1390
1391         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1392                                                   &pool_index);
1393         if (err)
1394                 return err;
1395
1396         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1397                 return -EINVAL;
1398
1399         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1400         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
1401                                            tc_index, pool_type,
1402                                            pool_index, threshold);
1403 }
1404
1405 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
1406                                                struct genl_info *info)
1407 {
1408         struct devlink *devlink = info->user_ptr[0];
1409         struct devlink_sb *devlink_sb = info->user_ptr[1];
1410         const struct devlink_ops *ops = devlink->ops;
1411
1412         if (ops && ops->sb_occ_snapshot)
1413                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
1414         return -EOPNOTSUPP;
1415 }
1416
1417 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
1418                                                 struct genl_info *info)
1419 {
1420         struct devlink *devlink = info->user_ptr[0];
1421         struct devlink_sb *devlink_sb = info->user_ptr[1];
1422         const struct devlink_ops *ops = devlink->ops;
1423
1424         if (ops && ops->sb_occ_max_clear)
1425                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
1426         return -EOPNOTSUPP;
1427 }
1428
1429 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
1430                                    enum devlink_command cmd, u32 portid,
1431                                    u32 seq, int flags)
1432 {
1433         const struct devlink_ops *ops = devlink->ops;
1434         u8 inline_mode, encap_mode;
1435         void *hdr;
1436         int err = 0;
1437         u16 mode;
1438
1439         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1440         if (!hdr)
1441                 return -EMSGSIZE;
1442
1443         err = devlink_nl_put_handle(msg, devlink);
1444         if (err)
1445                 goto nla_put_failure;
1446
1447         if (ops->eswitch_mode_get) {
1448                 err = ops->eswitch_mode_get(devlink, &mode);
1449                 if (err)
1450                         goto nla_put_failure;
1451                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
1452                 if (err)
1453                         goto nla_put_failure;
1454         }
1455
1456         if (ops->eswitch_inline_mode_get) {
1457                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
1458                 if (err)
1459                         goto nla_put_failure;
1460                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
1461                                  inline_mode);
1462                 if (err)
1463                         goto nla_put_failure;
1464         }
1465
1466         if (ops->eswitch_encap_mode_get) {
1467                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
1468                 if (err)
1469                         goto nla_put_failure;
1470                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
1471                 if (err)
1472                         goto nla_put_failure;
1473         }
1474
1475         genlmsg_end(msg, hdr);
1476         return 0;
1477
1478 nla_put_failure:
1479         genlmsg_cancel(msg, hdr);
1480         return err;
1481 }
1482
1483 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
1484                                            struct genl_info *info)
1485 {
1486         struct devlink *devlink = info->user_ptr[0];
1487         const struct devlink_ops *ops = devlink->ops;
1488         struct sk_buff *msg;
1489         int err;
1490
1491         if (!ops)
1492                 return -EOPNOTSUPP;
1493
1494         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1495         if (!msg)
1496                 return -ENOMEM;
1497
1498         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
1499                                       info->snd_portid, info->snd_seq, 0);
1500
1501         if (err) {
1502                 nlmsg_free(msg);
1503                 return err;
1504         }
1505
1506         return genlmsg_reply(msg, info);
1507 }
1508
1509 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
1510                                            struct genl_info *info)
1511 {
1512         struct devlink *devlink = info->user_ptr[0];
1513         const struct devlink_ops *ops = devlink->ops;
1514         u8 inline_mode, encap_mode;
1515         int err = 0;
1516         u16 mode;
1517
1518         if (!ops)
1519                 return -EOPNOTSUPP;
1520
1521         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
1522                 if (!ops->eswitch_mode_set)
1523                         return -EOPNOTSUPP;
1524                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
1525                 err = ops->eswitch_mode_set(devlink, mode);
1526                 if (err)
1527                         return err;
1528         }
1529
1530         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
1531                 if (!ops->eswitch_inline_mode_set)
1532                         return -EOPNOTSUPP;
1533                 inline_mode = nla_get_u8(
1534                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
1535                 err = ops->eswitch_inline_mode_set(devlink, inline_mode);
1536                 if (err)
1537                         return err;
1538         }
1539
1540         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
1541                 if (!ops->eswitch_encap_mode_set)
1542                         return -EOPNOTSUPP;
1543                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
1544                 err = ops->eswitch_encap_mode_set(devlink, encap_mode);
1545                 if (err)
1546                         return err;
1547         }
1548
1549         return 0;
1550 }
1551
1552 int devlink_dpipe_match_put(struct sk_buff *skb,
1553                             struct devlink_dpipe_match *match)
1554 {
1555         struct devlink_dpipe_header *header = match->header;
1556         struct devlink_dpipe_field *field = &header->fields[match->field_id];
1557         struct nlattr *match_attr;
1558
1559         match_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_MATCH);
1560         if (!match_attr)
1561                 return -EMSGSIZE;
1562
1563         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
1564             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
1565             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
1566             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
1567             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
1568                 goto nla_put_failure;
1569
1570         nla_nest_end(skb, match_attr);
1571         return 0;
1572
1573 nla_put_failure:
1574         nla_nest_cancel(skb, match_attr);
1575         return -EMSGSIZE;
1576 }
1577 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
1578
1579 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
1580                                      struct sk_buff *skb)
1581 {
1582         struct nlattr *matches_attr;
1583
1584         matches_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
1585         if (!matches_attr)
1586                 return -EMSGSIZE;
1587
1588         if (table->table_ops->matches_dump(table->priv, skb))
1589                 goto nla_put_failure;
1590
1591         nla_nest_end(skb, matches_attr);
1592         return 0;
1593
1594 nla_put_failure:
1595         nla_nest_cancel(skb, matches_attr);
1596         return -EMSGSIZE;
1597 }
1598
1599 int devlink_dpipe_action_put(struct sk_buff *skb,
1600                              struct devlink_dpipe_action *action)
1601 {
1602         struct devlink_dpipe_header *header = action->header;
1603         struct devlink_dpipe_field *field = &header->fields[action->field_id];
1604         struct nlattr *action_attr;
1605
1606         action_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_ACTION);
1607         if (!action_attr)
1608                 return -EMSGSIZE;
1609
1610         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
1611             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
1612             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
1613             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
1614             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
1615                 goto nla_put_failure;
1616
1617         nla_nest_end(skb, action_attr);
1618         return 0;
1619
1620 nla_put_failure:
1621         nla_nest_cancel(skb, action_attr);
1622         return -EMSGSIZE;
1623 }
1624 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
1625
1626 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
1627                                      struct sk_buff *skb)
1628 {
1629         struct nlattr *actions_attr;
1630
1631         actions_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
1632         if (!actions_attr)
1633                 return -EMSGSIZE;
1634
1635         if (table->table_ops->actions_dump(table->priv, skb))
1636                 goto nla_put_failure;
1637
1638         nla_nest_end(skb, actions_attr);
1639         return 0;
1640
1641 nla_put_failure:
1642         nla_nest_cancel(skb, actions_attr);
1643         return -EMSGSIZE;
1644 }
1645
1646 static int devlink_dpipe_table_put(struct sk_buff *skb,
1647                                    struct devlink_dpipe_table *table)
1648 {
1649         struct nlattr *table_attr;
1650         u64 table_size;
1651
1652         table_size = table->table_ops->size_get(table->priv);
1653         table_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE);
1654         if (!table_attr)
1655                 return -EMSGSIZE;
1656
1657         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
1658             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
1659                               DEVLINK_ATTR_PAD))
1660                 goto nla_put_failure;
1661         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
1662                        table->counters_enabled))
1663                 goto nla_put_failure;
1664
1665         if (devlink_dpipe_matches_put(table, skb))
1666                 goto nla_put_failure;
1667
1668         if (devlink_dpipe_actions_put(table, skb))
1669                 goto nla_put_failure;
1670
1671         nla_nest_end(skb, table_attr);
1672         return 0;
1673
1674 nla_put_failure:
1675         nla_nest_cancel(skb, table_attr);
1676         return -EMSGSIZE;
1677 }
1678
1679 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
1680                                             struct genl_info *info)
1681 {
1682         int err;
1683
1684         if (*pskb) {
1685                 err = genlmsg_reply(*pskb, info);
1686                 if (err)
1687                         return err;
1688         }
1689         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1690         if (!*pskb)
1691                 return -ENOMEM;
1692         return 0;
1693 }
1694
1695 static int devlink_dpipe_tables_fill(struct genl_info *info,
1696                                      enum devlink_command cmd, int flags,
1697                                      struct list_head *dpipe_tables,
1698                                      const char *table_name)
1699 {
1700         struct devlink *devlink = info->user_ptr[0];
1701         struct devlink_dpipe_table *table;
1702         struct nlattr *tables_attr;
1703         struct sk_buff *skb = NULL;
1704         struct nlmsghdr *nlh;
1705         bool incomplete;
1706         void *hdr;
1707         int i;
1708         int err;
1709
1710         table = list_first_entry(dpipe_tables,
1711                                  struct devlink_dpipe_table, list);
1712 start_again:
1713         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
1714         if (err)
1715                 return err;
1716
1717         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
1718                           &devlink_nl_family, NLM_F_MULTI, cmd);
1719         if (!hdr) {
1720                 nlmsg_free(skb);
1721                 return -EMSGSIZE;
1722         }
1723
1724         if (devlink_nl_put_handle(skb, devlink))
1725                 goto nla_put_failure;
1726         tables_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLES);
1727         if (!tables_attr)
1728                 goto nla_put_failure;
1729
1730         i = 0;
1731         incomplete = false;
1732         list_for_each_entry_from(table, dpipe_tables, list) {
1733                 if (!table_name) {
1734                         err = devlink_dpipe_table_put(skb, table);
1735                         if (err) {
1736                                 if (!i)
1737                                         goto err_table_put;
1738                                 incomplete = true;
1739                                 break;
1740                         }
1741                 } else {
1742                         if (!strcmp(table->name, table_name)) {
1743                                 err = devlink_dpipe_table_put(skb, table);
1744                                 if (err)
1745                                         break;
1746                         }
1747                 }
1748                 i++;
1749         }
1750
1751         nla_nest_end(skb, tables_attr);
1752         genlmsg_end(skb, hdr);
1753         if (incomplete)
1754                 goto start_again;
1755
1756 send_done:
1757         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
1758                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
1759         if (!nlh) {
1760                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
1761                 if (err)
1762                         goto err_skb_send_alloc;
1763                 goto send_done;
1764         }
1765
1766         return genlmsg_reply(skb, info);
1767
1768 nla_put_failure:
1769         err = -EMSGSIZE;
1770 err_table_put:
1771 err_skb_send_alloc:
1772         genlmsg_cancel(skb, hdr);
1773         nlmsg_free(skb);
1774         return err;
1775 }
1776
1777 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
1778                                           struct genl_info *info)
1779 {
1780         struct devlink *devlink = info->user_ptr[0];
1781         const char *table_name =  NULL;
1782
1783         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
1784                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
1785
1786         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
1787                                          &devlink->dpipe_table_list,
1788                                          table_name);
1789 }
1790
1791 static int devlink_dpipe_value_put(struct sk_buff *skb,
1792                                    struct devlink_dpipe_value *value)
1793 {
1794         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
1795                     value->value_size, value->value))
1796                 return -EMSGSIZE;
1797         if (value->mask)
1798                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
1799                             value->value_size, value->mask))
1800                         return -EMSGSIZE;
1801         if (value->mapping_valid)
1802                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
1803                                 value->mapping_value))
1804                         return -EMSGSIZE;
1805         return 0;
1806 }
1807
1808 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
1809                                           struct devlink_dpipe_value *value)
1810 {
1811         if (!value->action)
1812                 return -EINVAL;
1813         if (devlink_dpipe_action_put(skb, value->action))
1814                 return -EMSGSIZE;
1815         if (devlink_dpipe_value_put(skb, value))
1816                 return -EMSGSIZE;
1817         return 0;
1818 }
1819
1820 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
1821                                            struct devlink_dpipe_value *values,
1822                                            unsigned int values_count)
1823 {
1824         struct nlattr *action_attr;
1825         int i;
1826         int err;
1827
1828         for (i = 0; i < values_count; i++) {
1829                 action_attr = nla_nest_start(skb,
1830                                              DEVLINK_ATTR_DPIPE_ACTION_VALUE);
1831                 if (!action_attr)
1832                         return -EMSGSIZE;
1833                 err = devlink_dpipe_action_value_put(skb, &values[i]);
1834                 if (err)
1835                         goto err_action_value_put;
1836                 nla_nest_end(skb, action_attr);
1837         }
1838         return 0;
1839
1840 err_action_value_put:
1841         nla_nest_cancel(skb, action_attr);
1842         return err;
1843 }
1844
1845 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
1846                                          struct devlink_dpipe_value *value)
1847 {
1848         if (!value->match)
1849                 return -EINVAL;
1850         if (devlink_dpipe_match_put(skb, value->match))
1851                 return -EMSGSIZE;
1852         if (devlink_dpipe_value_put(skb, value))
1853                 return -EMSGSIZE;
1854         return 0;
1855 }
1856
1857 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
1858                                           struct devlink_dpipe_value *values,
1859                                           unsigned int values_count)
1860 {
1861         struct nlattr *match_attr;
1862         int i;
1863         int err;
1864
1865         for (i = 0; i < values_count; i++) {
1866                 match_attr = nla_nest_start(skb,
1867                                             DEVLINK_ATTR_DPIPE_MATCH_VALUE);
1868                 if (!match_attr)
1869                         return -EMSGSIZE;
1870                 err = devlink_dpipe_match_value_put(skb, &values[i]);
1871                 if (err)
1872                         goto err_match_value_put;
1873                 nla_nest_end(skb, match_attr);
1874         }
1875         return 0;
1876
1877 err_match_value_put:
1878         nla_nest_cancel(skb, match_attr);
1879         return err;
1880 }
1881
1882 static int devlink_dpipe_entry_put(struct sk_buff *skb,
1883                                    struct devlink_dpipe_entry *entry)
1884 {
1885         struct nlattr *entry_attr, *matches_attr, *actions_attr;
1886         int err;
1887
1888         entry_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_ENTRY);
1889         if (!entry_attr)
1890                 return  -EMSGSIZE;
1891
1892         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
1893                               DEVLINK_ATTR_PAD))
1894                 goto nla_put_failure;
1895         if (entry->counter_valid)
1896                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
1897                                       entry->counter, DEVLINK_ATTR_PAD))
1898                         goto nla_put_failure;
1899
1900         matches_attr = nla_nest_start(skb,
1901                                       DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
1902         if (!matches_attr)
1903                 goto nla_put_failure;
1904
1905         err = devlink_dpipe_match_values_put(skb, entry->match_values,
1906                                              entry->match_values_count);
1907         if (err) {
1908                 nla_nest_cancel(skb, matches_attr);
1909                 goto err_match_values_put;
1910         }
1911         nla_nest_end(skb, matches_attr);
1912
1913         actions_attr = nla_nest_start(skb,
1914                                       DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
1915         if (!actions_attr)
1916                 goto nla_put_failure;
1917
1918         err = devlink_dpipe_action_values_put(skb, entry->action_values,
1919                                               entry->action_values_count);
1920         if (err) {
1921                 nla_nest_cancel(skb, actions_attr);
1922                 goto err_action_values_put;
1923         }
1924         nla_nest_end(skb, actions_attr);
1925
1926         nla_nest_end(skb, entry_attr);
1927         return 0;
1928
1929 nla_put_failure:
1930         err = -EMSGSIZE;
1931 err_match_values_put:
1932 err_action_values_put:
1933         nla_nest_cancel(skb, entry_attr);
1934         return err;
1935 }
1936
1937 static struct devlink_dpipe_table *
1938 devlink_dpipe_table_find(struct list_head *dpipe_tables,
1939                          const char *table_name)
1940 {
1941         struct devlink_dpipe_table *table;
1942
1943         list_for_each_entry_rcu(table, dpipe_tables, list) {
1944                 if (!strcmp(table->name, table_name))
1945                         return table;
1946         }
1947         return NULL;
1948 }
1949
1950 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
1951 {
1952         struct devlink *devlink;
1953         int err;
1954
1955         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
1956                                                dump_ctx->info);
1957         if (err)
1958                 return err;
1959
1960         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
1961                                     dump_ctx->info->snd_portid,
1962                                     dump_ctx->info->snd_seq,
1963                                     &devlink_nl_family, NLM_F_MULTI,
1964                                     dump_ctx->cmd);
1965         if (!dump_ctx->hdr)
1966                 goto nla_put_failure;
1967
1968         devlink = dump_ctx->info->user_ptr[0];
1969         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
1970                 goto nla_put_failure;
1971         dump_ctx->nest = nla_nest_start(dump_ctx->skb,
1972                                         DEVLINK_ATTR_DPIPE_ENTRIES);
1973         if (!dump_ctx->nest)
1974                 goto nla_put_failure;
1975         return 0;
1976
1977 nla_put_failure:
1978         genlmsg_cancel(dump_ctx->skb, dump_ctx->hdr);
1979         nlmsg_free(dump_ctx->skb);
1980         return -EMSGSIZE;
1981 }
1982 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
1983
1984 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
1985                                    struct devlink_dpipe_entry *entry)
1986 {
1987         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
1988 }
1989 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
1990
1991 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
1992 {
1993         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
1994         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
1995         return 0;
1996 }
1997 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
1998
1999 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
2000
2001 {
2002         unsigned int value_count, value_index;
2003         struct devlink_dpipe_value *value;
2004
2005         value = entry->action_values;
2006         value_count = entry->action_values_count;
2007         for (value_index = 0; value_index < value_count; value_index++) {
2008                 kfree(value[value_index].value);
2009                 kfree(value[value_index].mask);
2010         }
2011
2012         value = entry->match_values;
2013         value_count = entry->match_values_count;
2014         for (value_index = 0; value_index < value_count; value_index++) {
2015                 kfree(value[value_index].value);
2016                 kfree(value[value_index].mask);
2017         }
2018 }
2019 EXPORT_SYMBOL(devlink_dpipe_entry_clear);
2020
2021 static int devlink_dpipe_entries_fill(struct genl_info *info,
2022                                       enum devlink_command cmd, int flags,
2023                                       struct devlink_dpipe_table *table)
2024 {
2025         struct devlink_dpipe_dump_ctx dump_ctx;
2026         struct nlmsghdr *nlh;
2027         int err;
2028
2029         dump_ctx.skb = NULL;
2030         dump_ctx.cmd = cmd;
2031         dump_ctx.info = info;
2032
2033         err = table->table_ops->entries_dump(table->priv,
2034                                              table->counters_enabled,
2035                                              &dump_ctx);
2036         if (err)
2037                 goto err_entries_dump;
2038
2039 send_done:
2040         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
2041                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2042         if (!nlh) {
2043                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
2044                 if (err)
2045                         goto err_skb_send_alloc;
2046                 goto send_done;
2047         }
2048         return genlmsg_reply(dump_ctx.skb, info);
2049
2050 err_entries_dump:
2051 err_skb_send_alloc:
2052         genlmsg_cancel(dump_ctx.skb, dump_ctx.hdr);
2053         nlmsg_free(dump_ctx.skb);
2054         return err;
2055 }
2056
2057 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
2058                                             struct genl_info *info)
2059 {
2060         struct devlink *devlink = info->user_ptr[0];
2061         struct devlink_dpipe_table *table;
2062         const char *table_name;
2063
2064         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
2065                 return -EINVAL;
2066
2067         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2068         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2069                                          table_name);
2070         if (!table)
2071                 return -EINVAL;
2072
2073         if (!table->table_ops->entries_dump)
2074                 return -EINVAL;
2075
2076         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
2077                                           0, table);
2078 }
2079
2080 static int devlink_dpipe_fields_put(struct sk_buff *skb,
2081                                     const struct devlink_dpipe_header *header)
2082 {
2083         struct devlink_dpipe_field *field;
2084         struct nlattr *field_attr;
2085         int i;
2086
2087         for (i = 0; i < header->fields_count; i++) {
2088                 field = &header->fields[i];
2089                 field_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_FIELD);
2090                 if (!field_attr)
2091                         return -EMSGSIZE;
2092                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
2093                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2094                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
2095                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
2096                         goto nla_put_failure;
2097                 nla_nest_end(skb, field_attr);
2098         }
2099         return 0;
2100
2101 nla_put_failure:
2102         nla_nest_cancel(skb, field_attr);
2103         return -EMSGSIZE;
2104 }
2105
2106 static int devlink_dpipe_header_put(struct sk_buff *skb,
2107                                     struct devlink_dpipe_header *header)
2108 {
2109         struct nlattr *fields_attr, *header_attr;
2110         int err;
2111
2112         header_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADER);
2113         if (!header_attr)
2114                 return -EMSGSIZE;
2115
2116         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
2117             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2118             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2119                 goto nla_put_failure;
2120
2121         fields_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
2122         if (!fields_attr)
2123                 goto nla_put_failure;
2124
2125         err = devlink_dpipe_fields_put(skb, header);
2126         if (err) {
2127                 nla_nest_cancel(skb, fields_attr);
2128                 goto nla_put_failure;
2129         }
2130         nla_nest_end(skb, fields_attr);
2131         nla_nest_end(skb, header_attr);
2132         return 0;
2133
2134 nla_put_failure:
2135         err = -EMSGSIZE;
2136         nla_nest_cancel(skb, header_attr);
2137         return err;
2138 }
2139
2140 static int devlink_dpipe_headers_fill(struct genl_info *info,
2141                                       enum devlink_command cmd, int flags,
2142                                       struct devlink_dpipe_headers *
2143                                       dpipe_headers)
2144 {
2145         struct devlink *devlink = info->user_ptr[0];
2146         struct nlattr *headers_attr;
2147         struct sk_buff *skb = NULL;
2148         struct nlmsghdr *nlh;
2149         void *hdr;
2150         int i, j;
2151         int err;
2152
2153         i = 0;
2154 start_again:
2155         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2156         if (err)
2157                 return err;
2158
2159         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
2160                           &devlink_nl_family, NLM_F_MULTI, cmd);
2161         if (!hdr) {
2162                 nlmsg_free(skb);
2163                 return -EMSGSIZE;
2164         }
2165
2166         if (devlink_nl_put_handle(skb, devlink))
2167                 goto nla_put_failure;
2168         headers_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADERS);
2169         if (!headers_attr)
2170                 goto nla_put_failure;
2171
2172         j = 0;
2173         for (; i < dpipe_headers->headers_count; i++) {
2174                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
2175                 if (err) {
2176                         if (!j)
2177                                 goto err_table_put;
2178                         break;
2179                 }
2180                 j++;
2181         }
2182         nla_nest_end(skb, headers_attr);
2183         genlmsg_end(skb, hdr);
2184         if (i != dpipe_headers->headers_count)
2185                 goto start_again;
2186
2187 send_done:
2188         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
2189                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2190         if (!nlh) {
2191                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2192                 if (err)
2193                         goto err_skb_send_alloc;
2194                 goto send_done;
2195         }
2196         return genlmsg_reply(skb, info);
2197
2198 nla_put_failure:
2199         err = -EMSGSIZE;
2200 err_table_put:
2201 err_skb_send_alloc:
2202         genlmsg_cancel(skb, hdr);
2203         nlmsg_free(skb);
2204         return err;
2205 }
2206
2207 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
2208                                             struct genl_info *info)
2209 {
2210         struct devlink *devlink = info->user_ptr[0];
2211
2212         if (!devlink->dpipe_headers)
2213                 return -EOPNOTSUPP;
2214         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
2215                                           0, devlink->dpipe_headers);
2216 }
2217
2218 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
2219                                             const char *table_name,
2220                                             bool enable)
2221 {
2222         struct devlink_dpipe_table *table;
2223
2224         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2225                                          table_name);
2226         if (!table)
2227                 return -EINVAL;
2228
2229         if (table->counter_control_extern)
2230                 return -EOPNOTSUPP;
2231
2232         if (!(table->counters_enabled ^ enable))
2233                 return 0;
2234
2235         table->counters_enabled = enable;
2236         if (table->table_ops->counters_set_update)
2237                 table->table_ops->counters_set_update(table->priv, enable);
2238         return 0;
2239 }
2240
2241 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
2242                                                    struct genl_info *info)
2243 {
2244         struct devlink *devlink = info->user_ptr[0];
2245         const char *table_name;
2246         bool counters_enable;
2247
2248         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
2249             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
2250                 return -EINVAL;
2251
2252         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2253         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
2254
2255         return devlink_dpipe_table_counters_set(devlink, table_name,
2256                                                 counters_enable);
2257 }
2258
2259 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
2260         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
2261         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
2262         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
2263         [DEVLINK_ATTR_PORT_TYPE] = { .type = NLA_U16 },
2264         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
2265         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
2266         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
2267         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
2268         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
2269         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
2270         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
2271         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
2272         [DEVLINK_ATTR_ESWITCH_MODE] = { .type = NLA_U16 },
2273         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
2274         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
2275         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
2276         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
2277 };
2278
2279 static const struct genl_ops devlink_nl_ops[] = {
2280         {
2281                 .cmd = DEVLINK_CMD_GET,
2282                 .doit = devlink_nl_cmd_get_doit,
2283                 .dumpit = devlink_nl_cmd_get_dumpit,
2284                 .policy = devlink_nl_policy,
2285                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2286                 /* can be retrieved by unprivileged users */
2287         },
2288         {
2289                 .cmd = DEVLINK_CMD_PORT_GET,
2290                 .doit = devlink_nl_cmd_port_get_doit,
2291                 .dumpit = devlink_nl_cmd_port_get_dumpit,
2292                 .policy = devlink_nl_policy,
2293                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
2294                 /* can be retrieved by unprivileged users */
2295         },
2296         {
2297                 .cmd = DEVLINK_CMD_PORT_SET,
2298                 .doit = devlink_nl_cmd_port_set_doit,
2299                 .policy = devlink_nl_policy,
2300                 .flags = GENL_ADMIN_PERM,
2301                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
2302         },
2303         {
2304                 .cmd = DEVLINK_CMD_PORT_SPLIT,
2305                 .doit = devlink_nl_cmd_port_split_doit,
2306                 .policy = devlink_nl_policy,
2307                 .flags = GENL_ADMIN_PERM,
2308                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2309         },
2310         {
2311                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
2312                 .doit = devlink_nl_cmd_port_unsplit_doit,
2313                 .policy = devlink_nl_policy,
2314                 .flags = GENL_ADMIN_PERM,
2315                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2316         },
2317         {
2318                 .cmd = DEVLINK_CMD_SB_GET,
2319                 .doit = devlink_nl_cmd_sb_get_doit,
2320                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
2321                 .policy = devlink_nl_policy,
2322                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2323                                   DEVLINK_NL_FLAG_NEED_SB,
2324                 /* can be retrieved by unprivileged users */
2325         },
2326         {
2327                 .cmd = DEVLINK_CMD_SB_POOL_GET,
2328                 .doit = devlink_nl_cmd_sb_pool_get_doit,
2329                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
2330                 .policy = devlink_nl_policy,
2331                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2332                                   DEVLINK_NL_FLAG_NEED_SB,
2333                 /* can be retrieved by unprivileged users */
2334         },
2335         {
2336                 .cmd = DEVLINK_CMD_SB_POOL_SET,
2337                 .doit = devlink_nl_cmd_sb_pool_set_doit,
2338                 .policy = devlink_nl_policy,
2339                 .flags = GENL_ADMIN_PERM,
2340                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2341                                   DEVLINK_NL_FLAG_NEED_SB,
2342         },
2343         {
2344                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
2345                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
2346                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
2347                 .policy = devlink_nl_policy,
2348                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2349                                   DEVLINK_NL_FLAG_NEED_SB,
2350                 /* can be retrieved by unprivileged users */
2351         },
2352         {
2353                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
2354                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
2355                 .policy = devlink_nl_policy,
2356                 .flags = GENL_ADMIN_PERM,
2357                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2358                                   DEVLINK_NL_FLAG_NEED_SB,
2359         },
2360         {
2361                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
2362                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
2363                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
2364                 .policy = devlink_nl_policy,
2365                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2366                                   DEVLINK_NL_FLAG_NEED_SB,
2367                 /* can be retrieved by unprivileged users */
2368         },
2369         {
2370                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
2371                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
2372                 .policy = devlink_nl_policy,
2373                 .flags = GENL_ADMIN_PERM,
2374                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2375                                   DEVLINK_NL_FLAG_NEED_SB,
2376         },
2377         {
2378                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
2379                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
2380                 .policy = devlink_nl_policy,
2381                 .flags = GENL_ADMIN_PERM,
2382                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2383                                   DEVLINK_NL_FLAG_NEED_SB |
2384                                   DEVLINK_NL_FLAG_LOCK_PORTS,
2385         },
2386         {
2387                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
2388                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
2389                 .policy = devlink_nl_policy,
2390                 .flags = GENL_ADMIN_PERM,
2391                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2392                                   DEVLINK_NL_FLAG_NEED_SB |
2393                                   DEVLINK_NL_FLAG_LOCK_PORTS,
2394         },
2395         {
2396                 .cmd = DEVLINK_CMD_ESWITCH_GET,
2397                 .doit = devlink_nl_cmd_eswitch_get_doit,
2398                 .policy = devlink_nl_policy,
2399                 .flags = GENL_ADMIN_PERM,
2400                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2401         },
2402         {
2403                 .cmd = DEVLINK_CMD_ESWITCH_SET,
2404                 .doit = devlink_nl_cmd_eswitch_set_doit,
2405                 .policy = devlink_nl_policy,
2406                 .flags = GENL_ADMIN_PERM,
2407                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2408         },
2409         {
2410                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
2411                 .doit = devlink_nl_cmd_dpipe_table_get,
2412                 .policy = devlink_nl_policy,
2413                 .flags = GENL_ADMIN_PERM,
2414                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2415         },
2416         {
2417                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
2418                 .doit = devlink_nl_cmd_dpipe_entries_get,
2419                 .policy = devlink_nl_policy,
2420                 .flags = GENL_ADMIN_PERM,
2421                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2422         },
2423         {
2424                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
2425                 .doit = devlink_nl_cmd_dpipe_headers_get,
2426                 .policy = devlink_nl_policy,
2427                 .flags = GENL_ADMIN_PERM,
2428                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2429         },
2430         {
2431                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
2432                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
2433                 .policy = devlink_nl_policy,
2434                 .flags = GENL_ADMIN_PERM,
2435                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2436         },
2437 };
2438
2439 static struct genl_family devlink_nl_family __ro_after_init = {
2440         .name           = DEVLINK_GENL_NAME,
2441         .version        = DEVLINK_GENL_VERSION,
2442         .maxattr        = DEVLINK_ATTR_MAX,
2443         .netnsok        = true,
2444         .pre_doit       = devlink_nl_pre_doit,
2445         .post_doit      = devlink_nl_post_doit,
2446         .module         = THIS_MODULE,
2447         .ops            = devlink_nl_ops,
2448         .n_ops          = ARRAY_SIZE(devlink_nl_ops),
2449         .mcgrps         = devlink_nl_mcgrps,
2450         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
2451 };
2452
2453 /**
2454  *      devlink_alloc - Allocate new devlink instance resources
2455  *
2456  *      @ops: ops
2457  *      @priv_size: size of user private data
2458  *
2459  *      Allocate new devlink instance resources, including devlink index
2460  *      and name.
2461  */
2462 struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
2463 {
2464         struct devlink *devlink;
2465
2466         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
2467         if (!devlink)
2468                 return NULL;
2469         devlink->ops = ops;
2470         devlink_net_set(devlink, &init_net);
2471         INIT_LIST_HEAD(&devlink->port_list);
2472         INIT_LIST_HEAD(&devlink->sb_list);
2473         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
2474         return devlink;
2475 }
2476 EXPORT_SYMBOL_GPL(devlink_alloc);
2477
2478 /**
2479  *      devlink_register - Register devlink instance
2480  *
2481  *      @devlink: devlink
2482  */
2483 int devlink_register(struct devlink *devlink, struct device *dev)
2484 {
2485         mutex_lock(&devlink_mutex);
2486         devlink->dev = dev;
2487         list_add_tail(&devlink->list, &devlink_list);
2488         devlink_notify(devlink, DEVLINK_CMD_NEW);
2489         mutex_unlock(&devlink_mutex);
2490         return 0;
2491 }
2492 EXPORT_SYMBOL_GPL(devlink_register);
2493
2494 /**
2495  *      devlink_unregister - Unregister devlink instance
2496  *
2497  *      @devlink: devlink
2498  */
2499 void devlink_unregister(struct devlink *devlink)
2500 {
2501         mutex_lock(&devlink_mutex);
2502         devlink_notify(devlink, DEVLINK_CMD_DEL);
2503         list_del(&devlink->list);
2504         mutex_unlock(&devlink_mutex);
2505 }
2506 EXPORT_SYMBOL_GPL(devlink_unregister);
2507
2508 /**
2509  *      devlink_free - Free devlink instance resources
2510  *
2511  *      @devlink: devlink
2512  */
2513 void devlink_free(struct devlink *devlink)
2514 {
2515         kfree(devlink);
2516 }
2517 EXPORT_SYMBOL_GPL(devlink_free);
2518
2519 /**
2520  *      devlink_port_register - Register devlink port
2521  *
2522  *      @devlink: devlink
2523  *      @devlink_port: devlink port
2524  *      @port_index
2525  *
2526  *      Register devlink port with provided port index. User can use
2527  *      any indexing, even hw-related one. devlink_port structure
2528  *      is convenient to be embedded inside user driver private structure.
2529  *      Note that the caller should take care of zeroing the devlink_port
2530  *      structure.
2531  */
2532 int devlink_port_register(struct devlink *devlink,
2533                           struct devlink_port *devlink_port,
2534                           unsigned int port_index)
2535 {
2536         mutex_lock(&devlink_port_mutex);
2537         if (devlink_port_index_exists(devlink, port_index)) {
2538                 mutex_unlock(&devlink_port_mutex);
2539                 return -EEXIST;
2540         }
2541         devlink_port->devlink = devlink;
2542         devlink_port->index = port_index;
2543         devlink_port->registered = true;
2544         list_add_tail(&devlink_port->list, &devlink->port_list);
2545         mutex_unlock(&devlink_port_mutex);
2546         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2547         return 0;
2548 }
2549 EXPORT_SYMBOL_GPL(devlink_port_register);
2550
2551 /**
2552  *      devlink_port_unregister - Unregister devlink port
2553  *
2554  *      @devlink_port: devlink port
2555  */
2556 void devlink_port_unregister(struct devlink_port *devlink_port)
2557 {
2558         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
2559         mutex_lock(&devlink_port_mutex);
2560         list_del(&devlink_port->list);
2561         mutex_unlock(&devlink_port_mutex);
2562 }
2563 EXPORT_SYMBOL_GPL(devlink_port_unregister);
2564
2565 static void __devlink_port_type_set(struct devlink_port *devlink_port,
2566                                     enum devlink_port_type type,
2567                                     void *type_dev)
2568 {
2569         devlink_port->type = type;
2570         devlink_port->type_dev = type_dev;
2571         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2572 }
2573
2574 /**
2575  *      devlink_port_type_eth_set - Set port type to Ethernet
2576  *
2577  *      @devlink_port: devlink port
2578  *      @netdev: related netdevice
2579  */
2580 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
2581                                struct net_device *netdev)
2582 {
2583         return __devlink_port_type_set(devlink_port,
2584                                        DEVLINK_PORT_TYPE_ETH, netdev);
2585 }
2586 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
2587
2588 /**
2589  *      devlink_port_type_ib_set - Set port type to InfiniBand
2590  *
2591  *      @devlink_port: devlink port
2592  *      @ibdev: related IB device
2593  */
2594 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
2595                               struct ib_device *ibdev)
2596 {
2597         return __devlink_port_type_set(devlink_port,
2598                                        DEVLINK_PORT_TYPE_IB, ibdev);
2599 }
2600 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
2601
2602 /**
2603  *      devlink_port_type_clear - Clear port type
2604  *
2605  *      @devlink_port: devlink port
2606  */
2607 void devlink_port_type_clear(struct devlink_port *devlink_port)
2608 {
2609         return __devlink_port_type_set(devlink_port,
2610                                        DEVLINK_PORT_TYPE_NOTSET, NULL);
2611 }
2612 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
2613
2614 /**
2615  *      devlink_port_split_set - Set port is split
2616  *
2617  *      @devlink_port: devlink port
2618  *      @split_group: split group - identifies group split port is part of
2619  */
2620 void devlink_port_split_set(struct devlink_port *devlink_port,
2621                             u32 split_group)
2622 {
2623         devlink_port->split = true;
2624         devlink_port->split_group = split_group;
2625         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2626 }
2627 EXPORT_SYMBOL_GPL(devlink_port_split_set);
2628
2629 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
2630                         u32 size, u16 ingress_pools_count,
2631                         u16 egress_pools_count, u16 ingress_tc_count,
2632                         u16 egress_tc_count)
2633 {
2634         struct devlink_sb *devlink_sb;
2635         int err = 0;
2636
2637         mutex_lock(&devlink_mutex);
2638         if (devlink_sb_index_exists(devlink, sb_index)) {
2639                 err = -EEXIST;
2640                 goto unlock;
2641         }
2642
2643         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
2644         if (!devlink_sb) {
2645                 err = -ENOMEM;
2646                 goto unlock;
2647         }
2648         devlink_sb->index = sb_index;
2649         devlink_sb->size = size;
2650         devlink_sb->ingress_pools_count = ingress_pools_count;
2651         devlink_sb->egress_pools_count = egress_pools_count;
2652         devlink_sb->ingress_tc_count = ingress_tc_count;
2653         devlink_sb->egress_tc_count = egress_tc_count;
2654         list_add_tail(&devlink_sb->list, &devlink->sb_list);
2655 unlock:
2656         mutex_unlock(&devlink_mutex);
2657         return err;
2658 }
2659 EXPORT_SYMBOL_GPL(devlink_sb_register);
2660
2661 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
2662 {
2663         struct devlink_sb *devlink_sb;
2664
2665         mutex_lock(&devlink_mutex);
2666         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
2667         WARN_ON(!devlink_sb);
2668         list_del(&devlink_sb->list);
2669         mutex_unlock(&devlink_mutex);
2670         kfree(devlink_sb);
2671 }
2672 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
2673
2674 /**
2675  *      devlink_dpipe_headers_register - register dpipe headers
2676  *
2677  *      @devlink: devlink
2678  *      @dpipe_headers: dpipe header array
2679  *
2680  *      Register the headers supported by hardware.
2681  */
2682 int devlink_dpipe_headers_register(struct devlink *devlink,
2683                                    struct devlink_dpipe_headers *dpipe_headers)
2684 {
2685         mutex_lock(&devlink_mutex);
2686         devlink->dpipe_headers = dpipe_headers;
2687         mutex_unlock(&devlink_mutex);
2688         return 0;
2689 }
2690 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
2691
2692 /**
2693  *      devlink_dpipe_headers_unregister - unregister dpipe headers
2694  *
2695  *      @devlink: devlink
2696  *
2697  *      Unregister the headers supported by hardware.
2698  */
2699 void devlink_dpipe_headers_unregister(struct devlink *devlink)
2700 {
2701         mutex_lock(&devlink_mutex);
2702         devlink->dpipe_headers = NULL;
2703         mutex_unlock(&devlink_mutex);
2704 }
2705 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
2706
2707 /**
2708  *      devlink_dpipe_table_counter_enabled - check if counter allocation
2709  *                                            required
2710  *      @devlink: devlink
2711  *      @table_name: tables name
2712  *
2713  *      Used by driver to check if counter allocation is required.
2714  *      After counter allocation is turned on the table entries
2715  *      are updated to include counter statistics.
2716  *
2717  *      After that point on the driver must respect the counter
2718  *      state so that each entry added to the table is added
2719  *      with a counter.
2720  */
2721 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
2722                                          const char *table_name)
2723 {
2724         struct devlink_dpipe_table *table;
2725         bool enabled;
2726
2727         rcu_read_lock();
2728         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2729                                          table_name);
2730         enabled = false;
2731         if (table)
2732                 enabled = table->counters_enabled;
2733         rcu_read_unlock();
2734         return enabled;
2735 }
2736 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
2737
2738 /**
2739  *      devlink_dpipe_table_register - register dpipe table
2740  *
2741  *      @devlink: devlink
2742  *      @table_name: table name
2743  *      @table_ops: table ops
2744  *      @priv: priv
2745  *      @counter_control_extern: external control for counters
2746  */
2747 int devlink_dpipe_table_register(struct devlink *devlink,
2748                                  const char *table_name,
2749                                  struct devlink_dpipe_table_ops *table_ops,
2750                                  void *priv, bool counter_control_extern)
2751 {
2752         struct devlink_dpipe_table *table;
2753
2754         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name))
2755                 return -EEXIST;
2756
2757         if (WARN_ON(!table_ops->size_get))
2758                 return -EINVAL;
2759
2760         table = kzalloc(sizeof(*table), GFP_KERNEL);
2761         if (!table)
2762                 return -ENOMEM;
2763
2764         table->name = table_name;
2765         table->table_ops = table_ops;
2766         table->priv = priv;
2767         table->counter_control_extern = counter_control_extern;
2768
2769         mutex_lock(&devlink_mutex);
2770         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
2771         mutex_unlock(&devlink_mutex);
2772         return 0;
2773 }
2774 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
2775
2776 /**
2777  *      devlink_dpipe_table_unregister - unregister dpipe table
2778  *
2779  *      @devlink: devlink
2780  *      @table_name: table name
2781  */
2782 void devlink_dpipe_table_unregister(struct devlink *devlink,
2783                                     const char *table_name)
2784 {
2785         struct devlink_dpipe_table *table;
2786
2787         mutex_lock(&devlink_mutex);
2788         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2789                                          table_name);
2790         if (!table)
2791                 goto unlock;
2792         list_del_rcu(&table->list);
2793         mutex_unlock(&devlink_mutex);
2794         kfree_rcu(table, rcu);
2795         return;
2796 unlock:
2797         mutex_unlock(&devlink_mutex);
2798 }
2799 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
2800
2801 static int __init devlink_module_init(void)
2802 {
2803         return genl_register_family(&devlink_nl_family);
2804 }
2805
2806 static void __exit devlink_module_exit(void)
2807 {
2808         genl_unregister_family(&devlink_nl_family);
2809 }
2810
2811 module_init(devlink_module_init);
2812 module_exit(devlink_module_exit);
2813
2814 MODULE_LICENSE("GPL v2");
2815 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2816 MODULE_DESCRIPTION("Network physical device Netlink interface");
2817 MODULE_ALIAS_GENL_FAMILY(DEVLINK_GENL_NAME);