net/core/devlink.c

   1 /*
   2  * net/core/devlink.c - Network physical/parent device Netlink interface
   3  *
   4  * Heavily inspired by net/wireless/
   5  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   6  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17 #include <linux/slab.h>
  18 #include <linux/gfp.h>
  19 #include <linux/device.h>
  20 #include <linux/list.h>
  21 #include <linux/netdevice.h>
  22 #include <rdma/ib_verbs.h>
  23 #include <net/netlink.h>
  24 #include <net/genetlink.h>
  25 #include <net/rtnetlink.h>
  26 #include <net/net_namespace.h>
  27 #include <net/sock.h>
  28 #include <net/devlink.h>
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/devlink.h>
  31
  32 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
  33         {
  34                 .name = "destination mac",
  35                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
  36                 .bitwidth = 48,
  37         },
  38 };
  39
  40 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
  41         .name = "ethernet",
  42         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
  43         .fields = devlink_dpipe_fields_ethernet,
  44         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
  45         .global = true,
  46 };
  47 EXPORT_SYMBOL(devlink_dpipe_header_ethernet);
  48
  49 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
  50         {
  51                 .name = "destination ip",
  52                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
  53                 .bitwidth = 32,
  54         },
  55 };
  56
  57 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
  58         .name = "ipv4",
  59         .id = DEVLINK_DPIPE_HEADER_IPV4,
  60         .fields = devlink_dpipe_fields_ipv4,
  61         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
  62         .global = true,
  63 };
  64 EXPORT_SYMBOL(devlink_dpipe_header_ipv4);
  65
  66 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
  67         {
  68                 .name = "destination ip",
  69                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
  70                 .bitwidth = 128,
  71         },
  72 };
  73
  74 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
  75         .name = "ipv6",
  76         .id = DEVLINK_DPIPE_HEADER_IPV6,
  77         .fields = devlink_dpipe_fields_ipv6,
  78         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
  79         .global = true,
  80 };
  81 EXPORT_SYMBOL(devlink_dpipe_header_ipv6);
  82
  83 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
  84
  85 static LIST_HEAD(devlink_list);
  86
  87 /* devlink_mutex
  88  *
  89  * An overall lock guarding every operation coming from userspace.
  90  * It also guards devlink devices list and it is taken when
  91  * driver registers/unregisters it.
  92  */
  93 static DEFINE_MUTEX(devlink_mutex);
  94
  95 /* devlink_port_mutex
  96  *
  97  * Shared lock to guard lists of ports in all devlink devices.
  98  */
  99 static DEFINE_MUTEX(devlink_port_mutex);
 100
 101 static struct net *devlink_net(const struct devlink *devlink)
 102 {
 103         return read_pnet(&devlink->_net);
 104 }
 105
 106 static void devlink_net_set(struct devlink *devlink, struct net *net)
 107 {
 108         write_pnet(&devlink->_net, net);
 109 }
 110
 111 static struct devlink *devlink_get_from_attrs(struct net *net,
 112                                               struct nlattr **attrs)
 113 {
 114         struct devlink *devlink;
 115         char *busname;
 116         char *devname;
 117
 118         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 119                 return ERR_PTR(-EINVAL);
 120
 121         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 122         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 123
 124         list_for_each_entry(devlink, &devlink_list, list) {
 125                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 126                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 127                     net_eq(devlink_net(devlink), net))
 128                         return devlink;
 129         }
 130
 131         return ERR_PTR(-ENODEV);
 132 }
 133
 134 static struct devlink *devlink_get_from_info(struct genl_info *info)
 135 {
 136         return devlink_get_from_attrs(genl_info_net(info), info->attrs);
 137 }
 138
 139 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 140                                                       int port_index)
 141 {
 142         struct devlink_port *devlink_port;
 143
 144         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 145                 if (devlink_port->index == port_index)
 146                         return devlink_port;
 147         }
 148         return NULL;
 149 }
 150
 151 static bool devlink_port_index_exists(struct devlink *devlink, int port_index)
 152 {
 153         return devlink_port_get_by_index(devlink, port_index);
 154 }
 155
 156 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 157                                                         struct nlattr **attrs)
 158 {
 159         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 160                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 161                 struct devlink_port *devlink_port;
 162
 163                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 164                 if (!devlink_port)
 165                         return ERR_PTR(-ENODEV);
 166                 return devlink_port;
 167         }
 168         return ERR_PTR(-EINVAL);
 169 }
 170
 171 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 172                                                        struct genl_info *info)
 173 {
 174         return devlink_port_get_from_attrs(devlink, info->attrs);
 175 }
 176
 177 struct devlink_sb {
 178         struct list_head list;
 179         unsigned int index;
 180         u32 size;
 181         u16 ingress_pools_count;
 182         u16 egress_pools_count;
 183         u16 ingress_tc_count;
 184         u16 egress_tc_count;
 185 };
 186
 187 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 188 {
 189         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 190 }
 191
 192 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 193                                                   unsigned int sb_index)
 194 {
 195         struct devlink_sb *devlink_sb;
 196
 197         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 198                 if (devlink_sb->index == sb_index)
 199                         return devlink_sb;
 200         }
 201         return NULL;
 202 }
 203
 204 static bool devlink_sb_index_exists(struct devlink *devlink,
 205                                     unsigned int sb_index)
 206 {
 207         return devlink_sb_get_by_index(devlink, sb_index);
 208 }
 209
 210 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 211                                                     struct nlattr **attrs)
 212 {
 213         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 214                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 215                 struct devlink_sb *devlink_sb;
 216
 217                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 218                 if (!devlink_sb)
 219                         return ERR_PTR(-ENODEV);
 220                 return devlink_sb;
 221         }
 222         return ERR_PTR(-EINVAL);
 223 }
 224
 225 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 226                                                    struct genl_info *info)
 227 {
 228         return devlink_sb_get_from_attrs(devlink, info->attrs);
 229 }
 230
 231 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 232                                                 struct nlattr **attrs,
 233                                                 u16 *p_pool_index)
 234 {
 235         u16 val;
 236
 237         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 238                 return -EINVAL;
 239
 240         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 241         if (val >= devlink_sb_pool_count(devlink_sb))
 242                 return -EINVAL;
 243         *p_pool_index = val;
 244         return 0;
 245 }
 246
 247 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 248                                                struct genl_info *info,
 249                                                u16 *p_pool_index)
 250 {
 251         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 252                                                     p_pool_index);
 253 }
 254
 255 static int
 256 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 257                                     enum devlink_sb_pool_type *p_pool_type)
 258 {
 259         u8 val;
 260
 261         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 262                 return -EINVAL;
 263
 264         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 265         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 266             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 267                 return -EINVAL;
 268         *p_pool_type = val;
 269         return 0;
 270 }
 271
 272 static int
 273 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 274                                    enum devlink_sb_pool_type *p_pool_type)
 275 {
 276         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 277 }
 278
 279 static int
 280 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 281                                   enum devlink_sb_threshold_type *p_th_type)
 282 {
 283         u8 val;
 284
 285         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 286                 return -EINVAL;
 287
 288         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 289         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 290             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 291                 return -EINVAL;
 292         *p_th_type = val;
 293         return 0;
 294 }
 295
 296 static int
 297 devlink_sb_th_type_get_from_info(struct genl_info *info,
 298                                  enum devlink_sb_threshold_type *p_th_type)
 299 {
 300         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 301 }
 302
 303 static int
 304 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 305                                    struct nlattr **attrs,
 306                                    enum devlink_sb_pool_type pool_type,
 307                                    u16 *p_tc_index)
 308 {
 309         u16 val;
 310
 311         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 312                 return -EINVAL;
 313
 314         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 315         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 316             val >= devlink_sb->ingress_tc_count)
 317                 return -EINVAL;
 318         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 319             val >= devlink_sb->egress_tc_count)
 320                 return -EINVAL;
 321         *p_tc_index = val;
 322         return 0;
 323 }
 324
 325 static int
 326 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 327                                   struct genl_info *info,
 328                                   enum devlink_sb_pool_type pool_type,
 329                                   u16 *p_tc_index)
 330 {
 331         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 332                                                   pool_type, p_tc_index);
 333 }
 334
 335 #define DEVLINK_NL_FLAG_NEED_DEVLINK    BIT(0)
 336 #define DEVLINK_NL_FLAG_NEED_PORT       BIT(1)
 337 #define DEVLINK_NL_FLAG_NEED_SB         BIT(2)
 338 #define DEVLINK_NL_FLAG_LOCK_PORTS      BIT(3)
 339         /* port is not needed but we need to ensure they don't
 340          * change in the middle of command
 341          */
 342
 343 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 344                                struct sk_buff *skb, struct genl_info *info)
 345 {
 346         struct devlink *devlink;
 347
 348         mutex_lock(&devlink_mutex);
 349         devlink = devlink_get_from_info(info);
 350         if (IS_ERR(devlink)) {
 351                 mutex_unlock(&devlink_mutex);
 352                 return PTR_ERR(devlink);
 353         }
 354         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK) {
 355                 info->user_ptr[0] = devlink;
 356         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 357                 struct devlink_port *devlink_port;
 358
 359                 mutex_lock(&devlink_port_mutex);
 360                 devlink_port = devlink_port_get_from_info(devlink, info);
 361                 if (IS_ERR(devlink_port)) {
 362                         mutex_unlock(&devlink_port_mutex);
 363                         mutex_unlock(&devlink_mutex);
 364                         return PTR_ERR(devlink_port);
 365                 }
 366                 info->user_ptr[0] = devlink_port;
 367         }
 368         if (ops->internal_flags & DEVLINK_NL_FLAG_LOCK_PORTS) {
 369                 mutex_lock(&devlink_port_mutex);
 370         }
 371         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_SB) {
 372                 struct devlink_sb *devlink_sb;
 373
 374                 devlink_sb = devlink_sb_get_from_info(devlink, info);
 375                 if (IS_ERR(devlink_sb)) {
 376                         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT)
 377                                 mutex_unlock(&devlink_port_mutex);
 378                         mutex_unlock(&devlink_mutex);
 379                         return PTR_ERR(devlink_sb);
 380                 }
 381                 info->user_ptr[1] = devlink_sb;
 382         }
 383         return 0;
 384 }
 385
 386 static void devlink_nl_post_doit(const struct genl_ops *ops,
 387                                  struct sk_buff *skb, struct genl_info *info)
 388 {
 389         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT ||
 390             ops->internal_flags & DEVLINK_NL_FLAG_LOCK_PORTS)
 391                 mutex_unlock(&devlink_port_mutex);
 392         mutex_unlock(&devlink_mutex);
 393 }
 394
 395 static struct genl_family devlink_nl_family;
 396
 397 enum devlink_multicast_groups {
 398         DEVLINK_MCGRP_CONFIG,
 399 };
 400
 401 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 402         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 403 };
 404
 405 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 406 {
 407         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 408                 return -EMSGSIZE;
 409         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 410                 return -EMSGSIZE;
 411         return 0;
 412 }
 413
 414 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 415                            enum devlink_command cmd, u32 portid,
 416                            u32 seq, int flags)
 417 {
 418         void *hdr;
 419
 420         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 421         if (!hdr)
 422                 return -EMSGSIZE;
 423
 424         if (devlink_nl_put_handle(msg, devlink))
 425                 goto nla_put_failure;
 426
 427         genlmsg_end(msg, hdr);
 428         return 0;
 429
 430 nla_put_failure:
 431         genlmsg_cancel(msg, hdr);
 432         return -EMSGSIZE;
 433 }
 434
 435 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 436 {
 437         struct sk_buff *msg;
 438         int err;
 439
 440         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 441
 442         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 443         if (!msg)
 444                 return;
 445
 446         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 447         if (err) {
 448                 nlmsg_free(msg);
 449                 return;
 450         }
 451
 452         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 453                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 454 }
 455
 456 static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
 457                                 struct devlink_port *devlink_port,
 458                                 enum devlink_command cmd, u32 portid,
 459                                 u32 seq, int flags)
 460 {
 461         void *hdr;
 462
 463         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 464         if (!hdr)
 465                 return -EMSGSIZE;
 466
 467         if (devlink_nl_put_handle(msg, devlink))
 468                 goto nla_put_failure;
 469         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
 470                 goto nla_put_failure;
 471         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
 472                 goto nla_put_failure;
 473         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
 474             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
 475                         devlink_port->desired_type))
 476                 goto nla_put_failure;
 477         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
 478                 struct net_device *netdev = devlink_port->type_dev;
 479
 480                 if (netdev &&
 481                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
 482                                  netdev->ifindex) ||
 483                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
 484                                     netdev->name)))
 485                         goto nla_put_failure;
 486         }
 487         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
 488                 struct ib_device *ibdev = devlink_port->type_dev;
 489
 490                 if (ibdev &&
 491                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
 492                                    ibdev->name))
 493                         goto nla_put_failure;
 494         }
 495         if (devlink_port->split &&
 496             nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
 497                         devlink_port->split_group))
 498                 goto nla_put_failure;
 499
 500         genlmsg_end(msg, hdr);
 501         return 0;
 502
 503 nla_put_failure:
 504         genlmsg_cancel(msg, hdr);
 505         return -EMSGSIZE;
 506 }
 507
 508 static void devlink_port_notify(struct devlink_port *devlink_port,
 509                                 enum devlink_command cmd)
 510 {
 511         struct devlink *devlink = devlink_port->devlink;
 512         struct sk_buff *msg;
 513         int err;
 514
 515         if (!devlink_port->registered)
 516                 return;
 517
 518         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
 519
 520         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 521         if (!msg)
 522                 return;
 523
 524         err = devlink_nl_port_fill(msg, devlink, devlink_port, cmd, 0, 0, 0);
 525         if (err) {
 526                 nlmsg_free(msg);
 527                 return;
 528         }
 529
 530         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 531                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 532 }
 533
 534 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
 535 {
 536         struct devlink *devlink = info->user_ptr[0];
 537         struct sk_buff *msg;
 538         int err;
 539
 540         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 541         if (!msg)
 542                 return -ENOMEM;
 543
 544         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 545                               info->snd_portid, info->snd_seq, 0);
 546         if (err) {
 547                 nlmsg_free(msg);
 548                 return err;
 549         }
 550
 551         return genlmsg_reply(msg, info);
 552 }
 553
 554 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
 555                                      struct netlink_callback *cb)
 556 {
 557         struct devlink *devlink;
 558         int start = cb->args[0];
 559         int idx = 0;
 560         int err;
 561
 562         mutex_lock(&devlink_mutex);
 563         list_for_each_entry(devlink, &devlink_list, list) {
 564                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 565                         continue;
 566                 if (idx < start) {
 567                         idx++;
 568                         continue;
 569                 }
 570                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 571                                       NETLINK_CB(cb->skb).portid,
 572                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
 573                 if (err)
 574                         goto out;
 575                 idx++;
 576         }
 577 out:
 578         mutex_unlock(&devlink_mutex);
 579
 580         cb->args[0] = idx;
 581         return msg->len;
 582 }
 583
 584 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
 585                                         struct genl_info *info)
 586 {
 587         struct devlink_port *devlink_port = info->user_ptr[0];
 588         struct devlink *devlink = devlink_port->devlink;
 589         struct sk_buff *msg;
 590         int err;
 591
 592         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 593         if (!msg)
 594                 return -ENOMEM;
 595
 596         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 597                                    DEVLINK_CMD_PORT_NEW,
 598                                    info->snd_portid, info->snd_seq, 0);
 599         if (err) {
 600                 nlmsg_free(msg);
 601                 return err;
 602         }
 603
 604         return genlmsg_reply(msg, info);
 605 }
 606
 607 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
 608                                           struct netlink_callback *cb)
 609 {
 610         struct devlink *devlink;
 611         struct devlink_port *devlink_port;
 612         int start = cb->args[0];
 613         int idx = 0;
 614         int err;
 615
 616         mutex_lock(&devlink_mutex);
 617         mutex_lock(&devlink_port_mutex);
 618         list_for_each_entry(devlink, &devlink_list, list) {
 619                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 620                         continue;
 621                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
 622                         if (idx < start) {
 623                                 idx++;
 624                                 continue;
 625                         }
 626                         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 627                                                    DEVLINK_CMD_NEW,
 628                                                    NETLINK_CB(cb->skb).portid,
 629                                                    cb->nlh->nlmsg_seq,
 630                                                    NLM_F_MULTI);
 631                         if (err)
 632                                 goto out;
 633                         idx++;
 634                 }
 635         }
 636 out:
 637         mutex_unlock(&devlink_port_mutex);
 638         mutex_unlock(&devlink_mutex);
 639
 640         cb->args[0] = idx;
 641         return msg->len;
 642 }
 643
 644 static int devlink_port_type_set(struct devlink *devlink,
 645                                  struct devlink_port *devlink_port,
 646                                  enum devlink_port_type port_type)
 647
 648 {
 649         int err;
 650
 651         if (devlink->ops && devlink->ops->port_type_set) {
 652                 if (port_type == DEVLINK_PORT_TYPE_NOTSET)
 653                         return -EINVAL;
 654                 if (port_type == devlink_port->type)
 655                         return 0;
 656                 err = devlink->ops->port_type_set(devlink_port, port_type);
 657                 if (err)
 658                         return err;
 659                 devlink_port->desired_type = port_type;
 660                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
 661                 return 0;
 662         }
 663         return -EOPNOTSUPP;
 664 }
 665
 666 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
 667                                         struct genl_info *info)
 668 {
 669         struct devlink_port *devlink_port = info->user_ptr[0];
 670         struct devlink *devlink = devlink_port->devlink;
 671         int err;
 672
 673         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
 674                 enum devlink_port_type port_type;
 675
 676                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
 677                 err = devlink_port_type_set(devlink, devlink_port, port_type);
 678                 if (err)
 679                         return err;
 680         }
 681         return 0;
 682 }
 683
 684 static int devlink_port_split(struct devlink *devlink,
 685                               u32 port_index, u32 count)
 686
 687 {
 688         if (devlink->ops && devlink->ops->port_split)
 689                 return devlink->ops->port_split(devlink, port_index, count);
 690         return -EOPNOTSUPP;
 691 }
 692
 693 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
 694                                           struct genl_info *info)
 695 {
 696         struct devlink *devlink = info->user_ptr[0];
 697         u32 port_index;
 698         u32 count;
 699
 700         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX] ||
 701             !info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
 702                 return -EINVAL;
 703
 704         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
 705         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
 706         return devlink_port_split(devlink, port_index, count);
 707 }
 708
 709 static int devlink_port_unsplit(struct devlink *devlink, u32 port_index)
 710
 711 {
 712         if (devlink->ops && devlink->ops->port_unsplit)
 713                 return devlink->ops->port_unsplit(devlink, port_index);
 714         return -EOPNOTSUPP;
 715 }
 716
 717 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
 718                                             struct genl_info *info)
 719 {
 720         struct devlink *devlink = info->user_ptr[0];
 721         u32 port_index;
 722
 723         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX])
 724                 return -EINVAL;
 725
 726         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
 727         return devlink_port_unsplit(devlink, port_index);
 728 }
 729
 730 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
 731                               struct devlink_sb *devlink_sb,
 732                               enum devlink_command cmd, u32 portid,
 733                               u32 seq, int flags)
 734 {
 735         void *hdr;
 736
 737         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 738         if (!hdr)
 739                 return -EMSGSIZE;
 740
 741         if (devlink_nl_put_handle(msg, devlink))
 742                 goto nla_put_failure;
 743         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
 744                 goto nla_put_failure;
 745         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
 746                 goto nla_put_failure;
 747         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
 748                         devlink_sb->ingress_pools_count))
 749                 goto nla_put_failure;
 750         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
 751                         devlink_sb->egress_pools_count))
 752                 goto nla_put_failure;
 753         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
 754                         devlink_sb->ingress_tc_count))
 755                 goto nla_put_failure;
 756         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
 757                         devlink_sb->egress_tc_count))
 758                 goto nla_put_failure;
 759
 760         genlmsg_end(msg, hdr);
 761         return 0;
 762
 763 nla_put_failure:
 764         genlmsg_cancel(msg, hdr);
 765         return -EMSGSIZE;
 766 }
 767
 768 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
 769                                       struct genl_info *info)
 770 {
 771         struct devlink *devlink = info->user_ptr[0];
 772         struct devlink_sb *devlink_sb = info->user_ptr[1];
 773         struct sk_buff *msg;
 774         int err;
 775
 776         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 777         if (!msg)
 778                 return -ENOMEM;
 779
 780         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
 781                                  DEVLINK_CMD_SB_NEW,
 782                                  info->snd_portid, info->snd_seq, 0);
 783         if (err) {
 784                 nlmsg_free(msg);
 785                 return err;
 786         }
 787
 788         return genlmsg_reply(msg, info);
 789 }
 790
 791 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
 792                                         struct netlink_callback *cb)
 793 {
 794         struct devlink *devlink;
 795         struct devlink_sb *devlink_sb;
 796         int start = cb->args[0];
 797         int idx = 0;
 798         int err;
 799
 800         mutex_lock(&devlink_mutex);
 801         list_for_each_entry(devlink, &devlink_list, list) {
 802                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 803                         continue;
 804                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 805                         if (idx < start) {
 806                                 idx++;
 807                                 continue;
 808                         }
 809                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
 810                                                  DEVLINK_CMD_SB_NEW,
 811                                                  NETLINK_CB(cb->skb).portid,
 812                                                  cb->nlh->nlmsg_seq,
 813                                                  NLM_F_MULTI);
 814                         if (err)
 815                                 goto out;
 816                         idx++;
 817                 }
 818         }
 819 out:
 820         mutex_unlock(&devlink_mutex);
 821
 822         cb->args[0] = idx;
 823         return msg->len;
 824 }
 825
 826 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
 827                                    struct devlink_sb *devlink_sb,
 828                                    u16 pool_index, enum devlink_command cmd,
 829                                    u32 portid, u32 seq, int flags)
 830 {
 831         struct devlink_sb_pool_info pool_info;
 832         void *hdr;
 833         int err;
 834
 835         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
 836                                         pool_index, &pool_info);
 837         if (err)
 838                 return err;
 839
 840         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 841         if (!hdr)
 842                 return -EMSGSIZE;
 843
 844         if (devlink_nl_put_handle(msg, devlink))
 845                 goto nla_put_failure;
 846         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
 847                 goto nla_put_failure;
 848         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
 849                 goto nla_put_failure;
 850         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
 851                 goto nla_put_failure;
 852         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
 853                 goto nla_put_failure;
 854         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
 855                        pool_info.threshold_type))
 856                 goto nla_put_failure;
 857
 858         genlmsg_end(msg, hdr);
 859         return 0;
 860
 861 nla_put_failure:
 862         genlmsg_cancel(msg, hdr);
 863         return -EMSGSIZE;
 864 }
 865
 866 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
 867                                            struct genl_info *info)
 868 {
 869         struct devlink *devlink = info->user_ptr[0];
 870         struct devlink_sb *devlink_sb = info->user_ptr[1];
 871         struct sk_buff *msg;
 872         u16 pool_index;
 873         int err;
 874
 875         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
 876                                                   &pool_index);
 877         if (err)
 878                 return err;
 879
 880         if (!devlink->ops || !devlink->ops->sb_pool_get)
 881                 return -EOPNOTSUPP;
 882
 883         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 884         if (!msg)
 885                 return -ENOMEM;
 886
 887         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
 888                                       DEVLINK_CMD_SB_POOL_NEW,
 889                                       info->snd_portid, info->snd_seq, 0);
 890         if (err) {
 891                 nlmsg_free(msg);
 892                 return err;
 893         }
 894
 895         return genlmsg_reply(msg, info);
 896 }
 897
 898 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
 899                                 struct devlink *devlink,
 900                                 struct devlink_sb *devlink_sb,
 901                                 u32 portid, u32 seq)
 902 {
 903         u16 pool_count = devlink_sb_pool_count(devlink_sb);
 904         u16 pool_index;
 905         int err;
 906
 907         for (pool_index = 0; pool_index < pool_count; pool_index++) {
 908                 if (*p_idx < start) {
 909                         (*p_idx)++;
 910                         continue;
 911                 }
 912                 err = devlink_nl_sb_pool_fill(msg, devlink,
 913                                               devlink_sb,
 914                                               pool_index,
 915                                               DEVLINK_CMD_SB_POOL_NEW,
 916                                               portid, seq, NLM_F_MULTI);
 917                 if (err)
 918                         return err;
 919                 (*p_idx)++;
 920         }
 921         return 0;
 922 }
 923
 924 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
 925                                              struct netlink_callback *cb)
 926 {
 927         struct devlink *devlink;
 928         struct devlink_sb *devlink_sb;
 929         int start = cb->args[0];
 930         int idx = 0;
 931         int err;
 932
 933         mutex_lock(&devlink_mutex);
 934         list_for_each_entry(devlink, &devlink_list, list) {
 935                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
 936                     !devlink->ops || !devlink->ops->sb_pool_get)
 937                         continue;
 938                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 939                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
 940                                                    devlink_sb,
 941                                                    NETLINK_CB(cb->skb).portid,
 942                                                    cb->nlh->nlmsg_seq);
 943                         if (err && err != -EOPNOTSUPP)
 944                                 goto out;
 945                 }
 946         }
 947 out:
 948         mutex_unlock(&devlink_mutex);
 949
 950         cb->args[0] = idx;
 951         return msg->len;
 952 }
 953
 954 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
 955                                u16 pool_index, u32 size,
 956                                enum devlink_sb_threshold_type threshold_type)
 957
 958 {
 959         const struct devlink_ops *ops = devlink->ops;
 960
 961         if (ops && ops->sb_pool_set)
 962                 return ops->sb_pool_set(devlink, sb_index, pool_index,
 963                                         size, threshold_type);
 964         return -EOPNOTSUPP;
 965 }
 966
 967 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
 968                                            struct genl_info *info)
 969 {
 970         struct devlink *devlink = info->user_ptr[0];
 971         struct devlink_sb *devlink_sb = info->user_ptr[1];
 972         enum devlink_sb_threshold_type threshold_type;
 973         u16 pool_index;
 974         u32 size;
 975         int err;
 976
 977         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
 978                                                   &pool_index);
 979         if (err)
 980                 return err;
 981
 982         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
 983         if (err)
 984                 return err;
 985
 986         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
 987                 return -EINVAL;
 988
 989         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
 990         return devlink_sb_pool_set(devlink, devlink_sb->index,
 991                                    pool_index, size, threshold_type);
 992 }
 993
 994 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
 995                                         struct devlink *devlink,
 996                                         struct devlink_port *devlink_port,
 997                                         struct devlink_sb *devlink_sb,
 998                                         u16 pool_index,
 999                                         enum devlink_command cmd,
1000                                         u32 portid, u32 seq, int flags)
1001 {
1002         const struct devlink_ops *ops = devlink->ops;
1003         u32 threshold;
1004         void *hdr;
1005         int err;
1006
1007         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
1008                                     pool_index, &threshold);
1009         if (err)
1010                 return err;
1011
1012         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1013         if (!hdr)
1014                 return -EMSGSIZE;
1015
1016         if (devlink_nl_put_handle(msg, devlink))
1017                 goto nla_put_failure;
1018         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1019                 goto nla_put_failure;
1020         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1021                 goto nla_put_failure;
1022         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1023                 goto nla_put_failure;
1024         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1025                 goto nla_put_failure;
1026
1027         if (ops->sb_occ_port_pool_get) {
1028                 u32 cur;
1029                 u32 max;
1030
1031                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
1032                                                 pool_index, &cur, &max);
1033                 if (err && err != -EOPNOTSUPP)
1034                         return err;
1035                 if (!err) {
1036                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1037                                 goto nla_put_failure;
1038                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1039                                 goto nla_put_failure;
1040                 }
1041         }
1042
1043         genlmsg_end(msg, hdr);
1044         return 0;
1045
1046 nla_put_failure:
1047         genlmsg_cancel(msg, hdr);
1048         return -EMSGSIZE;
1049 }
1050
1051 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
1052                                                 struct genl_info *info)
1053 {
1054         struct devlink_port *devlink_port = info->user_ptr[0];
1055         struct devlink *devlink = devlink_port->devlink;
1056         struct devlink_sb *devlink_sb = info->user_ptr[1];
1057         struct sk_buff *msg;
1058         u16 pool_index;
1059         int err;
1060
1061         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1062                                                   &pool_index);
1063         if (err)
1064                 return err;
1065
1066         if (!devlink->ops || !devlink->ops->sb_port_pool_get)
1067                 return -EOPNOTSUPP;
1068
1069         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1070         if (!msg)
1071                 return -ENOMEM;
1072
1073         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
1074                                            devlink_sb, pool_index,
1075                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1076                                            info->snd_portid, info->snd_seq, 0);
1077         if (err) {
1078                 nlmsg_free(msg);
1079                 return err;
1080         }
1081
1082         return genlmsg_reply(msg, info);
1083 }
1084
1085 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
1086                                      struct devlink *devlink,
1087                                      struct devlink_sb *devlink_sb,
1088                                      u32 portid, u32 seq)
1089 {
1090         struct devlink_port *devlink_port;
1091         u16 pool_count = devlink_sb_pool_count(devlink_sb);
1092         u16 pool_index;
1093         int err;
1094
1095         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1096                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
1097                         if (*p_idx < start) {
1098                                 (*p_idx)++;
1099                                 continue;
1100                         }
1101                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
1102                                                            devlink_port,
1103                                                            devlink_sb,
1104                                                            pool_index,
1105                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1106                                                            portid, seq,
1107                                                            NLM_F_MULTI);
1108                         if (err)
1109                                 return err;
1110                         (*p_idx)++;
1111                 }
1112         }
1113         return 0;
1114 }
1115
1116 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
1117                                                   struct netlink_callback *cb)
1118 {
1119         struct devlink *devlink;
1120         struct devlink_sb *devlink_sb;
1121         int start = cb->args[0];
1122         int idx = 0;
1123         int err;
1124
1125         mutex_lock(&devlink_mutex);
1126         mutex_lock(&devlink_port_mutex);
1127         list_for_each_entry(devlink, &devlink_list, list) {
1128                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1129                     !devlink->ops || !devlink->ops->sb_port_pool_get)
1130                         continue;
1131                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1132                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
1133                                                         devlink, devlink_sb,
1134                                                         NETLINK_CB(cb->skb).portid,
1135                                                         cb->nlh->nlmsg_seq);
1136                         if (err && err != -EOPNOTSUPP)
1137                                 goto out;
1138                 }
1139         }
1140 out:
1141         mutex_unlock(&devlink_port_mutex);
1142         mutex_unlock(&devlink_mutex);
1143
1144         cb->args[0] = idx;
1145         return msg->len;
1146 }
1147
1148 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
1149                                     unsigned int sb_index, u16 pool_index,
1150                                     u32 threshold)
1151
1152 {
1153         const struct devlink_ops *ops = devlink_port->devlink->ops;
1154
1155         if (ops && ops->sb_port_pool_set)
1156                 return ops->sb_port_pool_set(devlink_port, sb_index,
1157                                              pool_index, threshold);
1158         return -EOPNOTSUPP;
1159 }
1160
1161 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
1162                                                 struct genl_info *info)
1163 {
1164         struct devlink_port *devlink_port = info->user_ptr[0];
1165         struct devlink_sb *devlink_sb = info->user_ptr[1];
1166         u16 pool_index;
1167         u32 threshold;
1168         int err;
1169
1170         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1171                                                   &pool_index);
1172         if (err)
1173                 return err;
1174
1175         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1176                 return -EINVAL;
1177
1178         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1179         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
1180                                         pool_index, threshold);
1181 }
1182
1183 static int
1184 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
1185                                 struct devlink_port *devlink_port,
1186                                 struct devlink_sb *devlink_sb, u16 tc_index,
1187                                 enum devlink_sb_pool_type pool_type,
1188                                 enum devlink_command cmd,
1189                                 u32 portid, u32 seq, int flags)
1190 {
1191         const struct devlink_ops *ops = devlink->ops;
1192         u16 pool_index;
1193         u32 threshold;
1194         void *hdr;
1195         int err;
1196
1197         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
1198                                        tc_index, pool_type,
1199                                        &pool_index, &threshold);
1200         if (err)
1201                 return err;
1202
1203         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1204         if (!hdr)
1205                 return -EMSGSIZE;
1206
1207         if (devlink_nl_put_handle(msg, devlink))
1208                 goto nla_put_failure;
1209         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1210                 goto nla_put_failure;
1211         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1212                 goto nla_put_failure;
1213         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
1214                 goto nla_put_failure;
1215         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
1216                 goto nla_put_failure;
1217         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1218                 goto nla_put_failure;
1219         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1220                 goto nla_put_failure;
1221
1222         if (ops->sb_occ_tc_port_bind_get) {
1223                 u32 cur;
1224                 u32 max;
1225
1226                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
1227                                                    devlink_sb->index,
1228                                                    tc_index, pool_type,
1229                                                    &cur, &max);
1230                 if (err && err != -EOPNOTSUPP)
1231                         return err;
1232                 if (!err) {
1233                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1234                                 goto nla_put_failure;
1235                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1236                                 goto nla_put_failure;
1237                 }
1238         }
1239
1240         genlmsg_end(msg, hdr);
1241         return 0;
1242
1243 nla_put_failure:
1244         genlmsg_cancel(msg, hdr);
1245         return -EMSGSIZE;
1246 }
1247
1248 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
1249                                                    struct genl_info *info)
1250 {
1251         struct devlink_port *devlink_port = info->user_ptr[0];
1252         struct devlink *devlink = devlink_port->devlink;
1253         struct devlink_sb *devlink_sb = info->user_ptr[1];
1254         struct sk_buff *msg;
1255         enum devlink_sb_pool_type pool_type;
1256         u16 tc_index;
1257         int err;
1258
1259         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1260         if (err)
1261                 return err;
1262
1263         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1264                                                 pool_type, &tc_index);
1265         if (err)
1266                 return err;
1267
1268         if (!devlink->ops || !devlink->ops->sb_tc_pool_bind_get)
1269                 return -EOPNOTSUPP;
1270
1271         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1272         if (!msg)
1273                 return -ENOMEM;
1274
1275         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
1276                                               devlink_sb, tc_index, pool_type,
1277                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1278                                               info->snd_portid,
1279                                               info->snd_seq, 0);
1280         if (err) {
1281                 nlmsg_free(msg);
1282                 return err;
1283         }
1284
1285         return genlmsg_reply(msg, info);
1286 }
1287
1288 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1289                                         int start, int *p_idx,
1290                                         struct devlink *devlink,
1291                                         struct devlink_sb *devlink_sb,
1292                                         u32 portid, u32 seq)
1293 {
1294         struct devlink_port *devlink_port;
1295         u16 tc_index;
1296         int err;
1297
1298         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1299                 for (tc_index = 0;
1300                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
1301                         if (*p_idx < start) {
1302                                 (*p_idx)++;
1303                                 continue;
1304                         }
1305                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1306                                                               devlink_port,
1307                                                               devlink_sb,
1308                                                               tc_index,
1309                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
1310                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1311                                                               portid, seq,
1312                                                               NLM_F_MULTI);
1313                         if (err)
1314                                 return err;
1315                         (*p_idx)++;
1316                 }
1317                 for (tc_index = 0;
1318                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
1319                         if (*p_idx < start) {
1320                                 (*p_idx)++;
1321                                 continue;
1322                         }
1323                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1324                                                               devlink_port,
1325                                                               devlink_sb,
1326                                                               tc_index,
1327                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
1328                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1329                                                               portid, seq,
1330                                                               NLM_F_MULTI);
1331                         if (err)
1332                                 return err;
1333                         (*p_idx)++;
1334                 }
1335         }
1336         return 0;
1337 }
1338
1339 static int
1340 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1341                                           struct netlink_callback *cb)
1342 {
1343         struct devlink *devlink;
1344         struct devlink_sb *devlink_sb;
1345         int start = cb->args[0];
1346         int idx = 0;
1347         int err;
1348
1349         mutex_lock(&devlink_mutex);
1350         mutex_lock(&devlink_port_mutex);
1351         list_for_each_entry(devlink, &devlink_list, list) {
1352                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1353                     !devlink->ops || !devlink->ops->sb_tc_pool_bind_get)
1354                         continue;
1355                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1356                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
1357                                                            devlink,
1358                                                            devlink_sb,
1359                                                            NETLINK_CB(cb->skb).portid,
1360                                                            cb->nlh->nlmsg_seq);
1361                         if (err && err != -EOPNOTSUPP)
1362                                 goto out;
1363                 }
1364         }
1365 out:
1366         mutex_unlock(&devlink_port_mutex);
1367         mutex_unlock(&devlink_mutex);
1368
1369         cb->args[0] = idx;
1370         return msg->len;
1371 }
1372
1373 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1374                                        unsigned int sb_index, u16 tc_index,
1375                                        enum devlink_sb_pool_type pool_type,
1376                                        u16 pool_index, u32 threshold)
1377
1378 {
1379         const struct devlink_ops *ops = devlink_port->devlink->ops;
1380
1381         if (ops && ops->sb_tc_pool_bind_set)
1382                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
1383                                                 tc_index, pool_type,
1384                                                 pool_index, threshold);
1385         return -EOPNOTSUPP;
1386 }
1387
1388 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
1389                                                    struct genl_info *info)
1390 {
1391         struct devlink_port *devlink_port = info->user_ptr[0];
1392         struct devlink_sb *devlink_sb = info->user_ptr[1];
1393         enum devlink_sb_pool_type pool_type;
1394         u16 tc_index;
1395         u16 pool_index;
1396         u32 threshold;
1397         int err;
1398
1399         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1400         if (err)
1401                 return err;
1402
1403         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1404                                                 pool_type, &tc_index);
1405         if (err)
1406                 return err;
1407
1408         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1409                                                   &pool_index);
1410         if (err)
1411                 return err;
1412
1413         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1414                 return -EINVAL;
1415
1416         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1417         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
1418                                            tc_index, pool_type,
1419                                            pool_index, threshold);
1420 }
1421
1422 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
1423                                                struct genl_info *info)
1424 {
1425         struct devlink *devlink = info->user_ptr[0];
1426         struct devlink_sb *devlink_sb = info->user_ptr[1];
1427         const struct devlink_ops *ops = devlink->ops;
1428
1429         if (ops && ops->sb_occ_snapshot)
1430                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
1431         return -EOPNOTSUPP;
1432 }
1433
1434 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
1435                                                 struct genl_info *info)
1436 {
1437         struct devlink *devlink = info->user_ptr[0];
1438         struct devlink_sb *devlink_sb = info->user_ptr[1];
1439         const struct devlink_ops *ops = devlink->ops;
1440
1441         if (ops && ops->sb_occ_max_clear)
1442                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
1443         return -EOPNOTSUPP;
1444 }
1445
1446 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
1447                                    enum devlink_command cmd, u32 portid,
1448                                    u32 seq, int flags)
1449 {
1450         const struct devlink_ops *ops = devlink->ops;
1451         u8 inline_mode, encap_mode;
1452         void *hdr;
1453         int err = 0;
1454         u16 mode;
1455
1456         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1457         if (!hdr)
1458                 return -EMSGSIZE;
1459
1460         err = devlink_nl_put_handle(msg, devlink);
1461         if (err)
1462                 goto nla_put_failure;
1463
1464         if (ops->eswitch_mode_get) {
1465                 err = ops->eswitch_mode_get(devlink, &mode);
1466                 if (err)
1467                         goto nla_put_failure;
1468                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
1469                 if (err)
1470                         goto nla_put_failure;
1471         }
1472
1473         if (ops->eswitch_inline_mode_get) {
1474                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
1475                 if (err)
1476                         goto nla_put_failure;
1477                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
1478                                  inline_mode);
1479                 if (err)
1480                         goto nla_put_failure;
1481         }
1482
1483         if (ops->eswitch_encap_mode_get) {
1484                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
1485                 if (err)
1486                         goto nla_put_failure;
1487                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
1488                 if (err)
1489                         goto nla_put_failure;
1490         }
1491
1492         genlmsg_end(msg, hdr);
1493         return 0;
1494
1495 nla_put_failure:
1496         genlmsg_cancel(msg, hdr);
1497         return err;
1498 }
1499
1500 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
1501                                            struct genl_info *info)
1502 {
1503         struct devlink *devlink = info->user_ptr[0];
1504         const struct devlink_ops *ops = devlink->ops;
1505         struct sk_buff *msg;
1506         int err;
1507
1508         if (!ops)
1509                 return -EOPNOTSUPP;
1510
1511         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1512         if (!msg)
1513                 return -ENOMEM;
1514
1515         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
1516                                       info->snd_portid, info->snd_seq, 0);
1517
1518         if (err) {
1519                 nlmsg_free(msg);
1520                 return err;
1521         }
1522
1523         return genlmsg_reply(msg, info);
1524 }
1525
1526 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
1527                                            struct genl_info *info)
1528 {
1529         struct devlink *devlink = info->user_ptr[0];
1530         const struct devlink_ops *ops = devlink->ops;
1531         u8 inline_mode, encap_mode;
1532         int err = 0;
1533         u16 mode;
1534
1535         if (!ops)
1536                 return -EOPNOTSUPP;
1537
1538         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
1539                 if (!ops->eswitch_mode_set)
1540                         return -EOPNOTSUPP;
1541                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
1542                 err = ops->eswitch_mode_set(devlink, mode);
1543                 if (err)
1544                         return err;
1545         }
1546
1547         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
1548                 if (!ops->eswitch_inline_mode_set)
1549                         return -EOPNOTSUPP;
1550                 inline_mode = nla_get_u8(
1551                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
1552                 err = ops->eswitch_inline_mode_set(devlink, inline_mode);
1553                 if (err)
1554                         return err;
1555         }
1556
1557         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
1558                 if (!ops->eswitch_encap_mode_set)
1559                         return -EOPNOTSUPP;
1560                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
1561                 err = ops->eswitch_encap_mode_set(devlink, encap_mode);
1562                 if (err)
1563                         return err;
1564         }
1565
1566         return 0;
1567 }
1568
1569 int devlink_dpipe_match_put(struct sk_buff *skb,
1570                             struct devlink_dpipe_match *match)
1571 {
1572         struct devlink_dpipe_header *header = match->header;
1573         struct devlink_dpipe_field *field = &header->fields[match->field_id];
1574         struct nlattr *match_attr;
1575
1576         match_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_MATCH);
1577         if (!match_attr)
1578                 return -EMSGSIZE;
1579
1580         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
1581             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
1582             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
1583             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
1584             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
1585                 goto nla_put_failure;
1586
1587         nla_nest_end(skb, match_attr);
1588         return 0;
1589
1590 nla_put_failure:
1591         nla_nest_cancel(skb, match_attr);
1592         return -EMSGSIZE;
1593 }
1594 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
1595
1596 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
1597                                      struct sk_buff *skb)
1598 {
1599         struct nlattr *matches_attr;
1600
1601         matches_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
1602         if (!matches_attr)
1603                 return -EMSGSIZE;
1604
1605         if (table->table_ops->matches_dump(table->priv, skb))
1606                 goto nla_put_failure;
1607
1608         nla_nest_end(skb, matches_attr);
1609         return 0;
1610
1611 nla_put_failure:
1612         nla_nest_cancel(skb, matches_attr);
1613         return -EMSGSIZE;
1614 }
1615
1616 int devlink_dpipe_action_put(struct sk_buff *skb,
1617                              struct devlink_dpipe_action *action)
1618 {
1619         struct devlink_dpipe_header *header = action->header;
1620         struct devlink_dpipe_field *field = &header->fields[action->field_id];
1621         struct nlattr *action_attr;
1622
1623         action_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_ACTION);
1624         if (!action_attr)
1625                 return -EMSGSIZE;
1626
1627         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
1628             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
1629             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
1630             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
1631             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
1632                 goto nla_put_failure;
1633
1634         nla_nest_end(skb, action_attr);
1635         return 0;
1636
1637 nla_put_failure:
1638         nla_nest_cancel(skb, action_attr);
1639         return -EMSGSIZE;
1640 }
1641 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
1642
1643 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
1644                                      struct sk_buff *skb)
1645 {
1646         struct nlattr *actions_attr;
1647
1648         actions_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
1649         if (!actions_attr)
1650                 return -EMSGSIZE;
1651
1652         if (table->table_ops->actions_dump(table->priv, skb))
1653                 goto nla_put_failure;
1654
1655         nla_nest_end(skb, actions_attr);
1656         return 0;
1657
1658 nla_put_failure:
1659         nla_nest_cancel(skb, actions_attr);
1660         return -EMSGSIZE;
1661 }
1662
1663 static int devlink_dpipe_table_put(struct sk_buff *skb,
1664                                    struct devlink_dpipe_table *table)
1665 {
1666         struct nlattr *table_attr;
1667         u64 table_size;
1668
1669         table_size = table->table_ops->size_get(table->priv);
1670         table_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLE);
1671         if (!table_attr)
1672                 return -EMSGSIZE;
1673
1674         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
1675             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
1676                               DEVLINK_ATTR_PAD))
1677                 goto nla_put_failure;
1678         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
1679                        table->counters_enabled))
1680                 goto nla_put_failure;
1681
1682         if (devlink_dpipe_matches_put(table, skb))
1683                 goto nla_put_failure;
1684
1685         if (devlink_dpipe_actions_put(table, skb))
1686                 goto nla_put_failure;
1687
1688         nla_nest_end(skb, table_attr);
1689         return 0;
1690
1691 nla_put_failure:
1692         nla_nest_cancel(skb, table_attr);
1693         return -EMSGSIZE;
1694 }
1695
1696 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
1697                                             struct genl_info *info)
1698 {
1699         int err;
1700
1701         if (*pskb) {
1702                 err = genlmsg_reply(*pskb, info);
1703                 if (err)
1704                         return err;
1705         }
1706         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1707         if (!*pskb)
1708                 return -ENOMEM;
1709         return 0;
1710 }
1711
1712 static int devlink_dpipe_tables_fill(struct genl_info *info,
1713                                      enum devlink_command cmd, int flags,
1714                                      struct list_head *dpipe_tables,
1715                                      const char *table_name)
1716 {
1717         struct devlink *devlink = info->user_ptr[0];
1718         struct devlink_dpipe_table *table;
1719         struct nlattr *tables_attr;
1720         struct sk_buff *skb = NULL;
1721         struct nlmsghdr *nlh;
1722         bool incomplete;
1723         void *hdr;
1724         int i;
1725         int err;
1726
1727         table = list_first_entry(dpipe_tables,
1728                                  struct devlink_dpipe_table, list);
1729 start_again:
1730         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
1731         if (err)
1732                 return err;
1733
1734         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
1735                           &devlink_nl_family, NLM_F_MULTI, cmd);
1736         if (!hdr) {
1737                 nlmsg_free(skb);
1738                 return -EMSGSIZE;
1739         }
1740
1741         if (devlink_nl_put_handle(skb, devlink))
1742                 goto nla_put_failure;
1743         tables_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_TABLES);
1744         if (!tables_attr)
1745                 goto nla_put_failure;
1746
1747         i = 0;
1748         incomplete = false;
1749         list_for_each_entry_from(table, dpipe_tables, list) {
1750                 if (!table_name) {
1751                         err = devlink_dpipe_table_put(skb, table);
1752                         if (err) {
1753                                 if (!i)
1754                                         goto err_table_put;
1755                                 incomplete = true;
1756                                 break;
1757                         }
1758                 } else {
1759                         if (!strcmp(table->name, table_name)) {
1760                                 err = devlink_dpipe_table_put(skb, table);
1761                                 if (err)
1762                                         break;
1763                         }
1764                 }
1765                 i++;
1766         }
1767
1768         nla_nest_end(skb, tables_attr);
1769         genlmsg_end(skb, hdr);
1770         if (incomplete)
1771                 goto start_again;
1772
1773 send_done:
1774         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
1775                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
1776         if (!nlh) {
1777                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
1778                 if (err)
1779                         goto err_skb_send_alloc;
1780                 goto send_done;
1781         }
1782
1783         return genlmsg_reply(skb, info);
1784
1785 nla_put_failure:
1786         err = -EMSGSIZE;
1787 err_table_put:
1788 err_skb_send_alloc:
1789         genlmsg_cancel(skb, hdr);
1790         nlmsg_free(skb);
1791         return err;
1792 }
1793
1794 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
1795                                           struct genl_info *info)
1796 {
1797         struct devlink *devlink = info->user_ptr[0];
1798         const char *table_name =  NULL;
1799
1800         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
1801                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
1802
1803         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
1804                                          &devlink->dpipe_table_list,
1805                                          table_name);
1806 }
1807
1808 static int devlink_dpipe_value_put(struct sk_buff *skb,
1809                                    struct devlink_dpipe_value *value)
1810 {
1811         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
1812                     value->value_size, value->value))
1813                 return -EMSGSIZE;
1814         if (value->mask)
1815                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
1816                             value->value_size, value->mask))
1817                         return -EMSGSIZE;
1818         if (value->mapping_valid)
1819                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
1820                                 value->mapping_value))
1821                         return -EMSGSIZE;
1822         return 0;
1823 }
1824
1825 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
1826                                           struct devlink_dpipe_value *value)
1827 {
1828         if (!value->action)
1829                 return -EINVAL;
1830         if (devlink_dpipe_action_put(skb, value->action))
1831                 return -EMSGSIZE;
1832         if (devlink_dpipe_value_put(skb, value))
1833                 return -EMSGSIZE;
1834         return 0;
1835 }
1836
1837 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
1838                                            struct devlink_dpipe_value *values,
1839                                            unsigned int values_count)
1840 {
1841         struct nlattr *action_attr;
1842         int i;
1843         int err;
1844
1845         for (i = 0; i < values_count; i++) {
1846                 action_attr = nla_nest_start(skb,
1847                                              DEVLINK_ATTR_DPIPE_ACTION_VALUE);
1848                 if (!action_attr)
1849                         return -EMSGSIZE;
1850                 err = devlink_dpipe_action_value_put(skb, &values[i]);
1851                 if (err)
1852                         goto err_action_value_put;
1853                 nla_nest_end(skb, action_attr);
1854         }
1855         return 0;
1856
1857 err_action_value_put:
1858         nla_nest_cancel(skb, action_attr);
1859         return err;
1860 }
1861
1862 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
1863                                          struct devlink_dpipe_value *value)
1864 {
1865         if (!value->match)
1866                 return -EINVAL;
1867         if (devlink_dpipe_match_put(skb, value->match))
1868                 return -EMSGSIZE;
1869         if (devlink_dpipe_value_put(skb, value))
1870                 return -EMSGSIZE;
1871         return 0;
1872 }
1873
1874 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
1875                                           struct devlink_dpipe_value *values,
1876                                           unsigned int values_count)
1877 {
1878         struct nlattr *match_attr;
1879         int i;
1880         int err;
1881
1882         for (i = 0; i < values_count; i++) {
1883                 match_attr = nla_nest_start(skb,
1884                                             DEVLINK_ATTR_DPIPE_MATCH_VALUE);
1885                 if (!match_attr)
1886                         return -EMSGSIZE;
1887                 err = devlink_dpipe_match_value_put(skb, &values[i]);
1888                 if (err)
1889                         goto err_match_value_put;
1890                 nla_nest_end(skb, match_attr);
1891         }
1892         return 0;
1893
1894 err_match_value_put:
1895         nla_nest_cancel(skb, match_attr);
1896         return err;
1897 }
1898
1899 static int devlink_dpipe_entry_put(struct sk_buff *skb,
1900                                    struct devlink_dpipe_entry *entry)
1901 {
1902         struct nlattr *entry_attr, *matches_attr, *actions_attr;
1903         int err;
1904
1905         entry_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_ENTRY);
1906         if (!entry_attr)
1907                 return  -EMSGSIZE;
1908
1909         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
1910                               DEVLINK_ATTR_PAD))
1911                 goto nla_put_failure;
1912         if (entry->counter_valid)
1913                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
1914                                       entry->counter, DEVLINK_ATTR_PAD))
1915                         goto nla_put_failure;
1916
1917         matches_attr = nla_nest_start(skb,
1918                                       DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
1919         if (!matches_attr)
1920                 goto nla_put_failure;
1921
1922         err = devlink_dpipe_match_values_put(skb, entry->match_values,
1923                                              entry->match_values_count);
1924         if (err) {
1925                 nla_nest_cancel(skb, matches_attr);
1926                 goto err_match_values_put;
1927         }
1928         nla_nest_end(skb, matches_attr);
1929
1930         actions_attr = nla_nest_start(skb,
1931                                       DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
1932         if (!actions_attr)
1933                 goto nla_put_failure;
1934
1935         err = devlink_dpipe_action_values_put(skb, entry->action_values,
1936                                               entry->action_values_count);
1937         if (err) {
1938                 nla_nest_cancel(skb, actions_attr);
1939                 goto err_action_values_put;
1940         }
1941         nla_nest_end(skb, actions_attr);
1942
1943         nla_nest_end(skb, entry_attr);
1944         return 0;
1945
1946 nla_put_failure:
1947         err = -EMSGSIZE;
1948 err_match_values_put:
1949 err_action_values_put:
1950         nla_nest_cancel(skb, entry_attr);
1951         return err;
1952 }
1953
1954 static struct devlink_dpipe_table *
1955 devlink_dpipe_table_find(struct list_head *dpipe_tables,
1956                          const char *table_name)
1957 {
1958         struct devlink_dpipe_table *table;
1959
1960         list_for_each_entry_rcu(table, dpipe_tables, list) {
1961                 if (!strcmp(table->name, table_name))
1962                         return table;
1963         }
1964         return NULL;
1965 }
1966
1967 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
1968 {
1969         struct devlink *devlink;
1970         int err;
1971
1972         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
1973                                                dump_ctx->info);
1974         if (err)
1975                 return err;
1976
1977         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
1978                                     dump_ctx->info->snd_portid,
1979                                     dump_ctx->info->snd_seq,
1980                                     &devlink_nl_family, NLM_F_MULTI,
1981                                     dump_ctx->cmd);
1982         if (!dump_ctx->hdr)
1983                 goto nla_put_failure;
1984
1985         devlink = dump_ctx->info->user_ptr[0];
1986         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
1987                 goto nla_put_failure;
1988         dump_ctx->nest = nla_nest_start(dump_ctx->skb,
1989                                         DEVLINK_ATTR_DPIPE_ENTRIES);
1990         if (!dump_ctx->nest)
1991                 goto nla_put_failure;
1992         return 0;
1993
1994 nla_put_failure:
1995         genlmsg_cancel(dump_ctx->skb, dump_ctx->hdr);
1996         nlmsg_free(dump_ctx->skb);
1997         return -EMSGSIZE;
1998 }
1999 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
2000
2001 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
2002                                    struct devlink_dpipe_entry *entry)
2003 {
2004         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
2005 }
2006 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
2007
2008 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
2009 {
2010         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
2011         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
2012         return 0;
2013 }
2014 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
2015
2016 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
2017
2018 {
2019         unsigned int value_count, value_index;
2020         struct devlink_dpipe_value *value;
2021
2022         value = entry->action_values;
2023         value_count = entry->action_values_count;
2024         for (value_index = 0; value_index < value_count; value_index++) {
2025                 kfree(value[value_index].value);
2026                 kfree(value[value_index].mask);
2027         }
2028
2029         value = entry->match_values;
2030         value_count = entry->match_values_count;
2031         for (value_index = 0; value_index < value_count; value_index++) {
2032                 kfree(value[value_index].value);
2033                 kfree(value[value_index].mask);
2034         }
2035 }
2036 EXPORT_SYMBOL(devlink_dpipe_entry_clear);
2037
2038 static int devlink_dpipe_entries_fill(struct genl_info *info,
2039                                       enum devlink_command cmd, int flags,
2040                                       struct devlink_dpipe_table *table)
2041 {
2042         struct devlink_dpipe_dump_ctx dump_ctx;
2043         struct nlmsghdr *nlh;
2044         int err;
2045
2046         dump_ctx.skb = NULL;
2047         dump_ctx.cmd = cmd;
2048         dump_ctx.info = info;
2049
2050         err = table->table_ops->entries_dump(table->priv,
2051                                              table->counters_enabled,
2052                                              &dump_ctx);
2053         if (err)
2054                 goto err_entries_dump;
2055
2056 send_done:
2057         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
2058                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2059         if (!nlh) {
2060                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
2061                 if (err)
2062                         goto err_skb_send_alloc;
2063                 goto send_done;
2064         }
2065         return genlmsg_reply(dump_ctx.skb, info);
2066
2067 err_entries_dump:
2068 err_skb_send_alloc:
2069         genlmsg_cancel(dump_ctx.skb, dump_ctx.hdr);
2070         nlmsg_free(dump_ctx.skb);
2071         return err;
2072 }
2073
2074 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
2075                                             struct genl_info *info)
2076 {
2077         struct devlink *devlink = info->user_ptr[0];
2078         struct devlink_dpipe_table *table;
2079         const char *table_name;
2080
2081         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
2082                 return -EINVAL;
2083
2084         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2085         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2086                                          table_name);
2087         if (!table)
2088                 return -EINVAL;
2089
2090         if (!table->table_ops->entries_dump)
2091                 return -EINVAL;
2092
2093         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
2094                                           0, table);
2095 }
2096
2097 static int devlink_dpipe_fields_put(struct sk_buff *skb,
2098                                     const struct devlink_dpipe_header *header)
2099 {
2100         struct devlink_dpipe_field *field;
2101         struct nlattr *field_attr;
2102         int i;
2103
2104         for (i = 0; i < header->fields_count; i++) {
2105                 field = &header->fields[i];
2106                 field_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_FIELD);
2107                 if (!field_attr)
2108                         return -EMSGSIZE;
2109                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
2110                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2111                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
2112                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
2113                         goto nla_put_failure;
2114                 nla_nest_end(skb, field_attr);
2115         }
2116         return 0;
2117
2118 nla_put_failure:
2119         nla_nest_cancel(skb, field_attr);
2120         return -EMSGSIZE;
2121 }
2122
2123 static int devlink_dpipe_header_put(struct sk_buff *skb,
2124                                     struct devlink_dpipe_header *header)
2125 {
2126         struct nlattr *fields_attr, *header_attr;
2127         int err;
2128
2129         header_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADER);
2130         if (!header_attr)
2131                 return -EMSGSIZE;
2132
2133         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
2134             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2135             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2136                 goto nla_put_failure;
2137
2138         fields_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
2139         if (!fields_attr)
2140                 goto nla_put_failure;
2141
2142         err = devlink_dpipe_fields_put(skb, header);
2143         if (err) {
2144                 nla_nest_cancel(skb, fields_attr);
2145                 goto nla_put_failure;
2146         }
2147         nla_nest_end(skb, fields_attr);
2148         nla_nest_end(skb, header_attr);
2149         return 0;
2150
2151 nla_put_failure:
2152         err = -EMSGSIZE;
2153         nla_nest_cancel(skb, header_attr);
2154         return err;
2155 }
2156
2157 static int devlink_dpipe_headers_fill(struct genl_info *info,
2158                                       enum devlink_command cmd, int flags,
2159                                       struct devlink_dpipe_headers *
2160                                       dpipe_headers)
2161 {
2162         struct devlink *devlink = info->user_ptr[0];
2163         struct nlattr *headers_attr;
2164         struct sk_buff *skb = NULL;
2165         struct nlmsghdr *nlh;
2166         void *hdr;
2167         int i, j;
2168         int err;
2169
2170         i = 0;
2171 start_again:
2172         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2173         if (err)
2174                 return err;
2175
2176         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
2177                           &devlink_nl_family, NLM_F_MULTI, cmd);
2178         if (!hdr) {
2179                 nlmsg_free(skb);
2180                 return -EMSGSIZE;
2181         }
2182
2183         if (devlink_nl_put_handle(skb, devlink))
2184                 goto nla_put_failure;
2185         headers_attr = nla_nest_start(skb, DEVLINK_ATTR_DPIPE_HEADERS);
2186         if (!headers_attr)
2187                 goto nla_put_failure;
2188
2189         j = 0;
2190         for (; i < dpipe_headers->headers_count; i++) {
2191                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
2192                 if (err) {
2193                         if (!j)
2194                                 goto err_table_put;
2195                         break;
2196                 }
2197                 j++;
2198         }
2199         nla_nest_end(skb, headers_attr);
2200         genlmsg_end(skb, hdr);
2201         if (i != dpipe_headers->headers_count)
2202                 goto start_again;
2203
2204 send_done:
2205         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
2206                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2207         if (!nlh) {
2208                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2209                 if (err)
2210                         goto err_skb_send_alloc;
2211                 goto send_done;
2212         }
2213         return genlmsg_reply(skb, info);
2214
2215 nla_put_failure:
2216         err = -EMSGSIZE;
2217 err_table_put:
2218 err_skb_send_alloc:
2219         genlmsg_cancel(skb, hdr);
2220         nlmsg_free(skb);
2221         return err;
2222 }
2223
2224 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
2225                                             struct genl_info *info)
2226 {
2227         struct devlink *devlink = info->user_ptr[0];
2228
2229         if (!devlink->dpipe_headers)
2230                 return -EOPNOTSUPP;
2231         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
2232                                           0, devlink->dpipe_headers);
2233 }
2234
2235 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
2236                                             const char *table_name,
2237                                             bool enable)
2238 {
2239         struct devlink_dpipe_table *table;
2240
2241         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2242                                          table_name);
2243         if (!table)
2244                 return -EINVAL;
2245
2246         if (table->counter_control_extern)
2247                 return -EOPNOTSUPP;
2248
2249         if (!(table->counters_enabled ^ enable))
2250                 return 0;
2251
2252         table->counters_enabled = enable;
2253         if (table->table_ops->counters_set_update)
2254                 table->table_ops->counters_set_update(table->priv, enable);
2255         return 0;
2256 }
2257
2258 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
2259                                                    struct genl_info *info)
2260 {
2261         struct devlink *devlink = info->user_ptr[0];
2262         const char *table_name;
2263         bool counters_enable;
2264
2265         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
2266             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
2267                 return -EINVAL;
2268
2269         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2270         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
2271
2272         return devlink_dpipe_table_counters_set(devlink, table_name,
2273                                                 counters_enable);
2274 }
2275
2276 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
2277         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
2278         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
2279         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
2280         [DEVLINK_ATTR_PORT_TYPE] = { .type = NLA_U16 },
2281         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
2282         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
2283         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
2284         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
2285         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
2286         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
2287         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
2288         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
2289         [DEVLINK_ATTR_ESWITCH_MODE] = { .type = NLA_U16 },
2290         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
2291         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
2292         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
2293         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
2294 };
2295
2296 static const struct genl_ops devlink_nl_ops[] = {
2297         {
2298                 .cmd = DEVLINK_CMD_GET,
2299                 .doit = devlink_nl_cmd_get_doit,
2300                 .dumpit = devlink_nl_cmd_get_dumpit,
2301                 .policy = devlink_nl_policy,
2302                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2303                 /* can be retrieved by unprivileged users */
2304         },
2305         {
2306                 .cmd = DEVLINK_CMD_PORT_GET,
2307                 .doit = devlink_nl_cmd_port_get_doit,
2308                 .dumpit = devlink_nl_cmd_port_get_dumpit,
2309                 .policy = devlink_nl_policy,
2310                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
2311                 /* can be retrieved by unprivileged users */
2312         },
2313         {
2314                 .cmd = DEVLINK_CMD_PORT_SET,
2315                 .doit = devlink_nl_cmd_port_set_doit,
2316                 .policy = devlink_nl_policy,
2317                 .flags = GENL_ADMIN_PERM,
2318                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
2319         },
2320         {
2321                 .cmd = DEVLINK_CMD_PORT_SPLIT,
2322                 .doit = devlink_nl_cmd_port_split_doit,
2323                 .policy = devlink_nl_policy,
2324                 .flags = GENL_ADMIN_PERM,
2325                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2326         },
2327         {
2328                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
2329                 .doit = devlink_nl_cmd_port_unsplit_doit,
2330                 .policy = devlink_nl_policy,
2331                 .flags = GENL_ADMIN_PERM,
2332                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2333         },
2334         {
2335                 .cmd = DEVLINK_CMD_SB_GET,
2336                 .doit = devlink_nl_cmd_sb_get_doit,
2337                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
2338                 .policy = devlink_nl_policy,
2339                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2340                                   DEVLINK_NL_FLAG_NEED_SB,
2341                 /* can be retrieved by unprivileged users */
2342         },
2343         {
2344                 .cmd = DEVLINK_CMD_SB_POOL_GET,
2345                 .doit = devlink_nl_cmd_sb_pool_get_doit,
2346                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
2347                 .policy = devlink_nl_policy,
2348                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2349                                   DEVLINK_NL_FLAG_NEED_SB,
2350                 /* can be retrieved by unprivileged users */
2351         },
2352         {
2353                 .cmd = DEVLINK_CMD_SB_POOL_SET,
2354                 .doit = devlink_nl_cmd_sb_pool_set_doit,
2355                 .policy = devlink_nl_policy,
2356                 .flags = GENL_ADMIN_PERM,
2357                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2358                                   DEVLINK_NL_FLAG_NEED_SB,
2359         },
2360         {
2361                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
2362                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
2363                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
2364                 .policy = devlink_nl_policy,
2365                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2366                                   DEVLINK_NL_FLAG_NEED_SB,
2367                 /* can be retrieved by unprivileged users */
2368         },
2369         {
2370                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
2371                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
2372                 .policy = devlink_nl_policy,
2373                 .flags = GENL_ADMIN_PERM,
2374                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2375                                   DEVLINK_NL_FLAG_NEED_SB,
2376         },
2377         {
2378                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
2379                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
2380                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
2381                 .policy = devlink_nl_policy,
2382                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2383                                   DEVLINK_NL_FLAG_NEED_SB,
2384                 /* can be retrieved by unprivileged users */
2385         },
2386         {
2387                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
2388                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
2389                 .policy = devlink_nl_policy,
2390                 .flags = GENL_ADMIN_PERM,
2391                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT |
2392                                   DEVLINK_NL_FLAG_NEED_SB,
2393         },
2394         {
2395                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
2396                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
2397                 .policy = devlink_nl_policy,
2398                 .flags = GENL_ADMIN_PERM,
2399                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2400                                   DEVLINK_NL_FLAG_NEED_SB |
2401                                   DEVLINK_NL_FLAG_LOCK_PORTS,
2402         },
2403         {
2404                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
2405                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
2406                 .policy = devlink_nl_policy,
2407                 .flags = GENL_ADMIN_PERM,
2408                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
2409                                   DEVLINK_NL_FLAG_NEED_SB |
2410                                   DEVLINK_NL_FLAG_LOCK_PORTS,
2411         },
2412         {
2413                 .cmd = DEVLINK_CMD_ESWITCH_GET,
2414                 .doit = devlink_nl_cmd_eswitch_get_doit,
2415                 .policy = devlink_nl_policy,
2416                 .flags = GENL_ADMIN_PERM,
2417                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2418         },
2419         {
2420                 .cmd = DEVLINK_CMD_ESWITCH_SET,
2421                 .doit = devlink_nl_cmd_eswitch_set_doit,
2422                 .policy = devlink_nl_policy,
2423                 .flags = GENL_ADMIN_PERM,
2424                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2425         },
2426         {
2427                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
2428                 .doit = devlink_nl_cmd_dpipe_table_get,
2429                 .policy = devlink_nl_policy,
2430                 .flags = GENL_ADMIN_PERM,
2431                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2432         },
2433         {
2434                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
2435                 .doit = devlink_nl_cmd_dpipe_entries_get,
2436                 .policy = devlink_nl_policy,
2437                 .flags = GENL_ADMIN_PERM,
2438                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2439         },
2440         {
2441                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
2442                 .doit = devlink_nl_cmd_dpipe_headers_get,
2443                 .policy = devlink_nl_policy,
2444                 .flags = GENL_ADMIN_PERM,
2445                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2446         },
2447         {
2448                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
2449                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
2450                 .policy = devlink_nl_policy,
2451                 .flags = GENL_ADMIN_PERM,
2452                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
2453         },
2454 };
2455
2456 static struct genl_family devlink_nl_family __ro_after_init = {
2457         .name           = DEVLINK_GENL_NAME,
2458         .version        = DEVLINK_GENL_VERSION,
2459         .maxattr        = DEVLINK_ATTR_MAX,
2460         .netnsok        = true,
2461         .pre_doit       = devlink_nl_pre_doit,
2462         .post_doit      = devlink_nl_post_doit,
2463         .module         = THIS_MODULE,
2464         .ops            = devlink_nl_ops,
2465         .n_ops          = ARRAY_SIZE(devlink_nl_ops),
2466         .mcgrps         = devlink_nl_mcgrps,
2467         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
2468 };
2469
2470 /**
2471  *      devlink_alloc - Allocate new devlink instance resources
2472  *
2473  *      @ops: ops
2474  *      @priv_size: size of user private data
2475  *
2476  *      Allocate new devlink instance resources, including devlink index
2477  *      and name.
2478  */
2479 struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
2480 {
2481         struct devlink *devlink;
2482
2483         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
2484         if (!devlink)
2485                 return NULL;
2486         devlink->ops = ops;
2487         devlink_net_set(devlink, &init_net);
2488         INIT_LIST_HEAD(&devlink->port_list);
2489         INIT_LIST_HEAD(&devlink->sb_list);
2490         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
2491         return devlink;
2492 }
2493 EXPORT_SYMBOL_GPL(devlink_alloc);
2494
2495 /**
2496  *      devlink_register - Register devlink instance
2497  *
2498  *      @devlink: devlink
2499  */
2500 int devlink_register(struct devlink *devlink, struct device *dev)
2501 {
2502         mutex_lock(&devlink_mutex);
2503         devlink->dev = dev;
2504         list_add_tail(&devlink->list, &devlink_list);
2505         devlink_notify(devlink, DEVLINK_CMD_NEW);
2506         mutex_unlock(&devlink_mutex);
2507         return 0;
2508 }
2509 EXPORT_SYMBOL_GPL(devlink_register);
2510
2511 /**
2512  *      devlink_unregister - Unregister devlink instance
2513  *
2514  *      @devlink: devlink
2515  */
2516 void devlink_unregister(struct devlink *devlink)
2517 {
2518         mutex_lock(&devlink_mutex);
2519         devlink_notify(devlink, DEVLINK_CMD_DEL);
2520         list_del(&devlink->list);
2521         mutex_unlock(&devlink_mutex);
2522 }
2523 EXPORT_SYMBOL_GPL(devlink_unregister);
2524
2525 /**
2526  *      devlink_free - Free devlink instance resources
2527  *
2528  *      @devlink: devlink
2529  */
2530 void devlink_free(struct devlink *devlink)
2531 {
2532         kfree(devlink);
2533 }
2534 EXPORT_SYMBOL_GPL(devlink_free);
2535
2536 /**
2537  *      devlink_port_register - Register devlink port
2538  *
2539  *      @devlink: devlink
2540  *      @devlink_port: devlink port
2541  *      @port_index
2542  *
2543  *      Register devlink port with provided port index. User can use
2544  *      any indexing, even hw-related one. devlink_port structure
2545  *      is convenient to be embedded inside user driver private structure.
2546  *      Note that the caller should take care of zeroing the devlink_port
2547  *      structure.
2548  */
2549 int devlink_port_register(struct devlink *devlink,
2550                           struct devlink_port *devlink_port,
2551                           unsigned int port_index)
2552 {
2553         mutex_lock(&devlink_port_mutex);
2554         if (devlink_port_index_exists(devlink, port_index)) {
2555                 mutex_unlock(&devlink_port_mutex);
2556                 return -EEXIST;
2557         }
2558         devlink_port->devlink = devlink;
2559         devlink_port->index = port_index;
2560         devlink_port->registered = true;
2561         list_add_tail(&devlink_port->list, &devlink->port_list);
2562         mutex_unlock(&devlink_port_mutex);
2563         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2564         return 0;
2565 }
2566 EXPORT_SYMBOL_GPL(devlink_port_register);
2567
2568 /**
2569  *      devlink_port_unregister - Unregister devlink port
2570  *
2571  *      @devlink_port: devlink port
2572  */
2573 void devlink_port_unregister(struct devlink_port *devlink_port)
2574 {
2575         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
2576         mutex_lock(&devlink_port_mutex);
2577         list_del(&devlink_port->list);
2578         mutex_unlock(&devlink_port_mutex);
2579 }
2580 EXPORT_SYMBOL_GPL(devlink_port_unregister);
2581
2582 static void __devlink_port_type_set(struct devlink_port *devlink_port,
2583                                     enum devlink_port_type type,
2584                                     void *type_dev)
2585 {
2586         devlink_port->type = type;
2587         devlink_port->type_dev = type_dev;
2588         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2589 }
2590
2591 /**
2592  *      devlink_port_type_eth_set - Set port type to Ethernet
2593  *
2594  *      @devlink_port: devlink port
2595  *      @netdev: related netdevice
2596  */
2597 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
2598                                struct net_device *netdev)
2599 {
2600         return __devlink_port_type_set(devlink_port,
2601                                        DEVLINK_PORT_TYPE_ETH, netdev);
2602 }
2603 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
2604
2605 /**
2606  *      devlink_port_type_ib_set - Set port type to InfiniBand
2607  *
2608  *      @devlink_port: devlink port
2609  *      @ibdev: related IB device
2610  */
2611 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
2612                               struct ib_device *ibdev)
2613 {
2614         return __devlink_port_type_set(devlink_port,
2615                                        DEVLINK_PORT_TYPE_IB, ibdev);
2616 }
2617 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
2618
2619 /**
2620  *      devlink_port_type_clear - Clear port type
2621  *
2622  *      @devlink_port: devlink port
2623  */
2624 void devlink_port_type_clear(struct devlink_port *devlink_port)
2625 {
2626         return __devlink_port_type_set(devlink_port,
2627                                        DEVLINK_PORT_TYPE_NOTSET, NULL);
2628 }
2629 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
2630
2631 /**
2632  *      devlink_port_split_set - Set port is split
2633  *
2634  *      @devlink_port: devlink port
2635  *      @split_group: split group - identifies group split port is part of
2636  */
2637 void devlink_port_split_set(struct devlink_port *devlink_port,
2638                             u32 split_group)
2639 {
2640         devlink_port->split = true;
2641         devlink_port->split_group = split_group;
2642         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
2643 }
2644 EXPORT_SYMBOL_GPL(devlink_port_split_set);
2645
2646 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
2647                         u32 size, u16 ingress_pools_count,
2648                         u16 egress_pools_count, u16 ingress_tc_count,
2649                         u16 egress_tc_count)
2650 {
2651         struct devlink_sb *devlink_sb;
2652         int err = 0;
2653
2654         mutex_lock(&devlink_mutex);
2655         if (devlink_sb_index_exists(devlink, sb_index)) {
2656                 err = -EEXIST;
2657                 goto unlock;
2658         }
2659
2660         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
2661         if (!devlink_sb) {
2662                 err = -ENOMEM;
2663                 goto unlock;
2664         }
2665         devlink_sb->index = sb_index;
2666         devlink_sb->size = size;
2667         devlink_sb->ingress_pools_count = ingress_pools_count;
2668         devlink_sb->egress_pools_count = egress_pools_count;
2669         devlink_sb->ingress_tc_count = ingress_tc_count;
2670         devlink_sb->egress_tc_count = egress_tc_count;
2671         list_add_tail(&devlink_sb->list, &devlink->sb_list);
2672 unlock:
2673         mutex_unlock(&devlink_mutex);
2674         return err;
2675 }
2676 EXPORT_SYMBOL_GPL(devlink_sb_register);
2677
2678 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
2679 {
2680         struct devlink_sb *devlink_sb;
2681
2682         mutex_lock(&devlink_mutex);
2683         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
2684         WARN_ON(!devlink_sb);
2685         list_del(&devlink_sb->list);
2686         mutex_unlock(&devlink_mutex);
2687         kfree(devlink_sb);
2688 }
2689 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
2690
2691 /**
2692  *      devlink_dpipe_headers_register - register dpipe headers
2693  *
2694  *      @devlink: devlink
2695  *      @dpipe_headers: dpipe header array
2696  *
2697  *      Register the headers supported by hardware.
2698  */
2699 int devlink_dpipe_headers_register(struct devlink *devlink,
2700                                    struct devlink_dpipe_headers *dpipe_headers)
2701 {
2702         mutex_lock(&devlink_mutex);
2703         devlink->dpipe_headers = dpipe_headers;
2704         mutex_unlock(&devlink_mutex);
2705         return 0;
2706 }
2707 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
2708
2709 /**
2710  *      devlink_dpipe_headers_unregister - unregister dpipe headers
2711  *
2712  *      @devlink: devlink
2713  *
2714  *      Unregister the headers supported by hardware.
2715  */
2716 void devlink_dpipe_headers_unregister(struct devlink *devlink)
2717 {
2718         mutex_lock(&devlink_mutex);
2719         devlink->dpipe_headers = NULL;
2720         mutex_unlock(&devlink_mutex);
2721 }
2722 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
2723
2724 /**
2725  *      devlink_dpipe_table_counter_enabled - check if counter allocation
2726  *                                            required
2727  *      @devlink: devlink
2728  *      @table_name: tables name
2729  *
2730  *      Used by driver to check if counter allocation is required.
2731  *      After counter allocation is turned on the table entries
2732  *      are updated to include counter statistics.
2733  *
2734  *      After that point on the driver must respect the counter
2735  *      state so that each entry added to the table is added
2736  *      with a counter.
2737  */
2738 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
2739                                          const char *table_name)
2740 {
2741         struct devlink_dpipe_table *table;
2742         bool enabled;
2743
2744         rcu_read_lock();
2745         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2746                                          table_name);
2747         enabled = false;
2748         if (table)
2749                 enabled = table->counters_enabled;
2750         rcu_read_unlock();
2751         return enabled;
2752 }
2753 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
2754
2755 /**
2756  *      devlink_dpipe_table_register - register dpipe table
2757  *
2758  *      @devlink: devlink
2759  *      @table_name: table name
2760  *      @table_ops: table ops
2761  *      @priv: priv
2762  *      @counter_control_extern: external control for counters
2763  */
2764 int devlink_dpipe_table_register(struct devlink *devlink,
2765                                  const char *table_name,
2766                                  struct devlink_dpipe_table_ops *table_ops,
2767                                  void *priv, bool counter_control_extern)
2768 {
2769         struct devlink_dpipe_table *table;
2770
2771         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name))
2772                 return -EEXIST;
2773
2774         if (WARN_ON(!table_ops->size_get))
2775                 return -EINVAL;
2776
2777         table = kzalloc(sizeof(*table), GFP_KERNEL);
2778         if (!table)
2779                 return -ENOMEM;
2780
2781         table->name = table_name;
2782         table->table_ops = table_ops;
2783         table->priv = priv;
2784         table->counter_control_extern = counter_control_extern;
2785
2786         mutex_lock(&devlink_mutex);
2787         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
2788         mutex_unlock(&devlink_mutex);
2789         return 0;
2790 }
2791 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
2792
2793 /**
2794  *      devlink_dpipe_table_unregister - unregister dpipe table
2795  *
2796  *      @devlink: devlink
2797  *      @table_name: table name
2798  */
2799 void devlink_dpipe_table_unregister(struct devlink *devlink,
2800                                     const char *table_name)
2801 {
2802         struct devlink_dpipe_table *table;
2803
2804         mutex_lock(&devlink_mutex);
2805         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2806                                          table_name);
2807         if (!table)
2808                 goto unlock;
2809         list_del_rcu(&table->list);
2810         mutex_unlock(&devlink_mutex);
2811         kfree_rcu(table, rcu);
2812         return;
2813 unlock:
2814         mutex_unlock(&devlink_mutex);
2815 }
2816 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
2817
2818 static int __init devlink_module_init(void)
2819 {
2820         return genl_register_family(&devlink_nl_family);
2821 }
2822
2823 static void __exit devlink_module_exit(void)
2824 {
2825         genl_unregister_family(&devlink_nl_family);
2826 }
2827
2828 module_init(devlink_module_init);
2829 module_exit(devlink_module_exit);
2830
2831 MODULE_LICENSE("GPL v2");
2832 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2833 MODULE_DESCRIPTION("Network physical device Netlink interface");
2834 MODULE_ALIAS_GENL_FAMILY(DEVLINK_GENL_NAME);