drivers/net/ethernet/mellanox/mlxsw/core.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/device.h>
   7 #include <linux/export.h>
   8 #include <linux/err.h>
   9 #include <linux/if_link.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/completion.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/types.h>
  15 #include <linux/string.h>
  16 #include <linux/gfp.h>
  17 #include <linux/random.h>
  18 #include <linux/jiffies.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rcupdate.h>
  21 #include <linux/slab.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/firmware.h>
  24 #include <asm/byteorder.h>
  25 #include <net/devlink.h>
  26 #include <trace/events/devlink.h>
  27
  28 #include "core.h"
  29 #include "core_env.h"
  30 #include "item.h"
  31 #include "cmd.h"
  32 #include "port.h"
  33 #include "trap.h"
  34 #include "emad.h"
  35 #include "reg.h"
  36 #include "resources.h"
  37 #include "../mlxfw/mlxfw.h"
  38
  39 static LIST_HEAD(mlxsw_core_driver_list);
  40 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
  41
  42 static const char mlxsw_core_driver_name[] = "mlxsw_core";
  43
  44 static struct workqueue_struct *mlxsw_wq;
  45 static struct workqueue_struct *mlxsw_owq;
  46
  47 struct mlxsw_core_port {
  48         struct devlink_port devlink_port;
  49         void *port_driver_priv;
  50         u8 local_port;
  51 };
  52
  53 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
  54 {
  55         return mlxsw_core_port->port_driver_priv;
  56 }
  57 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
  58
  59 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
  60 {
  61         return mlxsw_core_port->port_driver_priv != NULL;
  62 }
  63
  64 struct mlxsw_core {
  65         struct mlxsw_driver *driver;
  66         const struct mlxsw_bus *bus;
  67         void *bus_priv;
  68         const struct mlxsw_bus_info *bus_info;
  69         struct workqueue_struct *emad_wq;
  70         struct list_head rx_listener_list;
  71         struct list_head event_listener_list;
  72         struct {
  73                 atomic64_t tid;
  74                 struct list_head trans_list;
  75                 spinlock_t trans_list_lock; /* protects trans_list writes */
  76                 bool use_emad;
  77                 bool enable_string_tlv;
  78         } emad;
  79         struct {
  80                 u8 *mapping; /* lag_id+port_index to local_port mapping */
  81         } lag;
  82         struct mlxsw_res res;
  83         struct mlxsw_hwmon *hwmon;
  84         struct mlxsw_thermal *thermal;
  85         struct mlxsw_core_port *ports;
  86         unsigned int max_ports;
  87         atomic_t active_ports_count;
  88         bool fw_flash_in_progress;
  89         struct {
  90                 struct devlink_health_reporter *fw_fatal;
  91         } health;
  92         struct mlxsw_env *env;
  93         bool is_initialized; /* Denotes if core was already initialized. */
  94         unsigned long driver_priv[];
  95         /* driver_priv has to be always the last item */
  96 };
  97
  98 #define MLXSW_PORT_MAX_PORTS_DEFAULT    0x40
  99
 100 static u64 mlxsw_ports_occ_get(void *priv)
 101 {
 102         struct mlxsw_core *mlxsw_core = priv;
 103
 104         return atomic_read(&mlxsw_core->active_ports_count);
 105 }
 106
 107 static int mlxsw_core_resources_ports_register(struct mlxsw_core *mlxsw_core)
 108 {
 109         struct devlink *devlink = priv_to_devlink(mlxsw_core);
 110         struct devlink_resource_size_params ports_num_params;
 111         u32 max_ports;
 112
 113         max_ports = mlxsw_core->max_ports - 1;
 114         devlink_resource_size_params_init(&ports_num_params, max_ports,
 115                                           max_ports, 1,
 116                                           DEVLINK_RESOURCE_UNIT_ENTRY);
 117
 118         return devlink_resource_register(devlink,
 119                                          DEVLINK_RESOURCE_GENERIC_NAME_PORTS,
 120                                          max_ports, MLXSW_CORE_RESOURCE_PORTS,
 121                                          DEVLINK_RESOURCE_ID_PARENT_TOP,
 122                                          &ports_num_params);
 123 }
 124
 125 static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core, bool reload)
 126 {
 127         struct devlink *devlink = priv_to_devlink(mlxsw_core);
 128         int err;
 129
 130         /* Switch ports are numbered from 1 to queried value */
 131         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_SYSTEM_PORT))
 132                 mlxsw_core->max_ports = MLXSW_CORE_RES_GET(mlxsw_core,
 133                                                            MAX_SYSTEM_PORT) + 1;
 134         else
 135                 mlxsw_core->max_ports = MLXSW_PORT_MAX_PORTS_DEFAULT + 1;
 136
 137         mlxsw_core->ports = kcalloc(mlxsw_core->max_ports,
 138                                     sizeof(struct mlxsw_core_port), GFP_KERNEL);
 139         if (!mlxsw_core->ports)
 140                 return -ENOMEM;
 141
 142         if (!reload) {
 143                 err = mlxsw_core_resources_ports_register(mlxsw_core);
 144                 if (err)
 145                         goto err_resources_ports_register;
 146         }
 147         atomic_set(&mlxsw_core->active_ports_count, 0);
 148         devlink_resource_occ_get_register(devlink, MLXSW_CORE_RESOURCE_PORTS,
 149                                           mlxsw_ports_occ_get, mlxsw_core);
 150
 151         return 0;
 152
 153 err_resources_ports_register:
 154         kfree(mlxsw_core->ports);
 155         return err;
 156 }
 157
 158 static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core, bool reload)
 159 {
 160         struct devlink *devlink = priv_to_devlink(mlxsw_core);
 161
 162         devlink_resource_occ_get_unregister(devlink, MLXSW_CORE_RESOURCE_PORTS);
 163         if (!reload)
 164                 devlink_resources_unregister(priv_to_devlink(mlxsw_core), NULL);
 165
 166         kfree(mlxsw_core->ports);
 167 }
 168
 169 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
 170 {
 171         return mlxsw_core->max_ports;
 172 }
 173 EXPORT_SYMBOL(mlxsw_core_max_ports);
 174
 175 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 176 {
 177         return mlxsw_core->driver_priv;
 178 }
 179 EXPORT_SYMBOL(mlxsw_core_driver_priv);
 180
 181 bool mlxsw_core_res_query_enabled(const struct mlxsw_core *mlxsw_core)
 182 {
 183         return mlxsw_core->driver->res_query_enabled;
 184 }
 185 EXPORT_SYMBOL(mlxsw_core_res_query_enabled);
 186
 187 bool mlxsw_core_temp_warn_enabled(const struct mlxsw_core *mlxsw_core)
 188 {
 189         return mlxsw_core->driver->temp_warn_enabled;
 190 }
 191
 192 bool
 193 mlxsw_core_fw_rev_minor_subminor_validate(const struct mlxsw_fw_rev *rev,
 194                                           const struct mlxsw_fw_rev *req_rev)
 195 {
 196         return rev->minor > req_rev->minor ||
 197                (rev->minor == req_rev->minor &&
 198                 rev->subminor >= req_rev->subminor);
 199 }
 200 EXPORT_SYMBOL(mlxsw_core_fw_rev_minor_subminor_validate);
 201
 202 struct mlxsw_rx_listener_item {
 203         struct list_head list;
 204         struct mlxsw_rx_listener rxl;
 205         void *priv;
 206         bool enabled;
 207 };
 208
 209 struct mlxsw_event_listener_item {
 210         struct list_head list;
 211         struct mlxsw_core *mlxsw_core;
 212         struct mlxsw_event_listener el;
 213         void *priv;
 214 };
 215
 216 /******************
 217  * EMAD processing
 218  ******************/
 219
 220 /* emad_eth_hdr_dmac
 221  * Destination MAC in EMAD's Ethernet header.
 222  * Must be set to 01:02:c9:00:00:01
 223  */
 224 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
 225
 226 /* emad_eth_hdr_smac
 227  * Source MAC in EMAD's Ethernet header.
 228  * Must be set to 00:02:c9:01:02:03
 229  */
 230 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
 231
 232 /* emad_eth_hdr_ethertype
 233  * Ethertype in EMAD's Ethernet header.
 234  * Must be set to 0x8932
 235  */
 236 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
 237
 238 /* emad_eth_hdr_mlx_proto
 239  * Mellanox protocol.
 240  * Must be set to 0x0.
 241  */
 242 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
 243
 244 /* emad_eth_hdr_ver
 245  * Mellanox protocol version.
 246  * Must be set to 0x0.
 247  */
 248 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
 249
 250 /* emad_op_tlv_type
 251  * Type of the TLV.
 252  * Must be set to 0x1 (operation TLV).
 253  */
 254 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
 255
 256 /* emad_op_tlv_len
 257  * Length of the operation TLV in u32.
 258  * Must be set to 0x4.
 259  */
 260 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
 261
 262 /* emad_op_tlv_dr
 263  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
 264  * EMAD. DR TLV must follow.
 265  *
 266  * Note: Currently not supported and must not be set.
 267  */
 268 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
 269
 270 /* emad_op_tlv_status
 271  * Returned status in case of EMAD response. Must be set to 0 in case
 272  * of EMAD request.
 273  * 0x0 - success
 274  * 0x1 - device is busy. Requester should retry
 275  * 0x2 - Mellanox protocol version not supported
 276  * 0x3 - unknown TLV
 277  * 0x4 - register not supported
 278  * 0x5 - operation class not supported
 279  * 0x6 - EMAD method not supported
 280  * 0x7 - bad parameter (e.g. port out of range)
 281  * 0x8 - resource not available
 282  * 0x9 - message receipt acknowledgment. Requester should retry
 283  * 0x70 - internal error
 284  */
 285 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
 286
 287 /* emad_op_tlv_register_id
 288  * Register ID of register within register TLV.
 289  */
 290 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
 291
 292 /* emad_op_tlv_r
 293  * Response bit. Setting to 1 indicates Response, otherwise request.
 294  */
 295 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
 296
 297 /* emad_op_tlv_method
 298  * EMAD method type.
 299  * 0x1 - query
 300  * 0x2 - write
 301  * 0x3 - send (currently not supported)
 302  * 0x4 - event
 303  */
 304 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
 305
 306 /* emad_op_tlv_class
 307  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
 308  */
 309 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
 310
 311 /* emad_op_tlv_tid
 312  * EMAD transaction ID. Used for pairing request and response EMADs.
 313  */
 314 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
 315
 316 /* emad_string_tlv_type
 317  * Type of the TLV.
 318  * Must be set to 0x2 (string TLV).
 319  */
 320 MLXSW_ITEM32(emad, string_tlv, type, 0x00, 27, 5);
 321
 322 /* emad_string_tlv_len
 323  * Length of the string TLV in u32.
 324  */
 325 MLXSW_ITEM32(emad, string_tlv, len, 0x00, 16, 11);
 326
 327 #define MLXSW_EMAD_STRING_TLV_STRING_LEN 128
 328
 329 /* emad_string_tlv_string
 330  * String provided by the device's firmware in case of erroneous register access
 331  */
 332 MLXSW_ITEM_BUF(emad, string_tlv, string, 0x04,
 333                MLXSW_EMAD_STRING_TLV_STRING_LEN);
 334
 335 /* emad_reg_tlv_type
 336  * Type of the TLV.
 337  * Must be set to 0x3 (register TLV).
 338  */
 339 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
 340
 341 /* emad_reg_tlv_len
 342  * Length of the operation TLV in u32.
 343  */
 344 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
 345
 346 /* emad_end_tlv_type
 347  * Type of the TLV.
 348  * Must be set to 0x0 (end TLV).
 349  */
 350 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
 351
 352 /* emad_end_tlv_len
 353  * Length of the end TLV in u32.
 354  * Must be set to 1.
 355  */
 356 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
 357
 358 enum mlxsw_core_reg_access_type {
 359         MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
 360         MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
 361 };
 362
 363 static inline const char *
 364 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
 365 {
 366         switch (type) {
 367         case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
 368                 return "query";
 369         case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
 370                 return "write";
 371         }
 372         BUG();
 373 }
 374
 375 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
 376 {
 377         mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
 378         mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
 379 }
 380
 381 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
 382                                     const struct mlxsw_reg_info *reg,
 383                                     char *payload)
 384 {
 385         mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
 386         mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
 387         memcpy(reg_tlv + sizeof(u32), payload, reg->len);
 388 }
 389
 390 static void mlxsw_emad_pack_string_tlv(char *string_tlv)
 391 {
 392         mlxsw_emad_string_tlv_type_set(string_tlv, MLXSW_EMAD_TLV_TYPE_STRING);
 393         mlxsw_emad_string_tlv_len_set(string_tlv, MLXSW_EMAD_STRING_TLV_LEN);
 394 }
 395
 396 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
 397                                    const struct mlxsw_reg_info *reg,
 398                                    enum mlxsw_core_reg_access_type type,
 399                                    u64 tid)
 400 {
 401         mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
 402         mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
 403         mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
 404         mlxsw_emad_op_tlv_status_set(op_tlv, 0);
 405         mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
 406         mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
 407         if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
 408                 mlxsw_emad_op_tlv_method_set(op_tlv,
 409                                              MLXSW_EMAD_OP_TLV_METHOD_QUERY);
 410         else
 411                 mlxsw_emad_op_tlv_method_set(op_tlv,
 412                                              MLXSW_EMAD_OP_TLV_METHOD_WRITE);
 413         mlxsw_emad_op_tlv_class_set(op_tlv,
 414                                     MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
 415         mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
 416 }
 417
 418 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
 419 {
 420         char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
 421
 422         mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
 423         mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
 424         mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
 425         mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
 426         mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
 427
 428         skb_reset_mac_header(skb);
 429
 430         return 0;
 431 }
 432
 433 static void mlxsw_emad_construct(struct sk_buff *skb,
 434                                  const struct mlxsw_reg_info *reg,
 435                                  char *payload,
 436                                  enum mlxsw_core_reg_access_type type,
 437                                  u64 tid, bool enable_string_tlv)
 438 {
 439         char *buf;
 440
 441         buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
 442         mlxsw_emad_pack_end_tlv(buf);
 443
 444         buf = skb_push(skb, reg->len + sizeof(u32));
 445         mlxsw_emad_pack_reg_tlv(buf, reg, payload);
 446
 447         if (enable_string_tlv) {
 448                 buf = skb_push(skb, MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32));
 449                 mlxsw_emad_pack_string_tlv(buf);
 450         }
 451
 452         buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
 453         mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
 454
 455         mlxsw_emad_construct_eth_hdr(skb);
 456 }
 457
 458 struct mlxsw_emad_tlv_offsets {
 459         u16 op_tlv;
 460         u16 string_tlv;
 461         u16 reg_tlv;
 462 };
 463
 464 static bool mlxsw_emad_tlv_is_string_tlv(const char *tlv)
 465 {
 466         u8 tlv_type = mlxsw_emad_string_tlv_type_get(tlv);
 467
 468         return tlv_type == MLXSW_EMAD_TLV_TYPE_STRING;
 469 }
 470
 471 static void mlxsw_emad_tlv_parse(struct sk_buff *skb)
 472 {
 473         struct mlxsw_emad_tlv_offsets *offsets =
 474                 (struct mlxsw_emad_tlv_offsets *) skb->cb;
 475
 476         offsets->op_tlv = MLXSW_EMAD_ETH_HDR_LEN;
 477         offsets->string_tlv = 0;
 478         offsets->reg_tlv = MLXSW_EMAD_ETH_HDR_LEN +
 479                            MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
 480
 481         /* If string TLV is present, it must come after the operation TLV. */
 482         if (mlxsw_emad_tlv_is_string_tlv(skb->data + offsets->reg_tlv)) {
 483                 offsets->string_tlv = offsets->reg_tlv;
 484                 offsets->reg_tlv += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
 485         }
 486 }
 487
 488 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
 489 {
 490         struct mlxsw_emad_tlv_offsets *offsets =
 491                 (struct mlxsw_emad_tlv_offsets *) skb->cb;
 492
 493         return ((char *) (skb->data + offsets->op_tlv));
 494 }
 495
 496 static char *mlxsw_emad_string_tlv(const struct sk_buff *skb)
 497 {
 498         struct mlxsw_emad_tlv_offsets *offsets =
 499                 (struct mlxsw_emad_tlv_offsets *) skb->cb;
 500
 501         if (!offsets->string_tlv)
 502                 return NULL;
 503
 504         return ((char *) (skb->data + offsets->string_tlv));
 505 }
 506
 507 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
 508 {
 509         struct mlxsw_emad_tlv_offsets *offsets =
 510                 (struct mlxsw_emad_tlv_offsets *) skb->cb;
 511
 512         return ((char *) (skb->data + offsets->reg_tlv));
 513 }
 514
 515 static char *mlxsw_emad_reg_payload(const char *reg_tlv)
 516 {
 517         return ((char *) (reg_tlv + sizeof(u32)));
 518 }
 519
 520 static char *mlxsw_emad_reg_payload_cmd(const char *mbox)
 521 {
 522         return ((char *) (mbox + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
 523 }
 524
 525 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
 526 {
 527         char *op_tlv;
 528
 529         op_tlv = mlxsw_emad_op_tlv(skb);
 530         return mlxsw_emad_op_tlv_tid_get(op_tlv);
 531 }
 532
 533 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
 534 {
 535         char *op_tlv;
 536
 537         op_tlv = mlxsw_emad_op_tlv(skb);
 538         return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
 539 }
 540
 541 static int mlxsw_emad_process_status(char *op_tlv,
 542                                      enum mlxsw_emad_op_tlv_status *p_status)
 543 {
 544         *p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
 545
 546         switch (*p_status) {
 547         case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
 548                 return 0;
 549         case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
 550         case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
 551                 return -EAGAIN;
 552         case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
 553         case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
 554         case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
 555         case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
 556         case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
 557         case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
 558         case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
 559         case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
 560         default:
 561                 return -EIO;
 562         }
 563 }
 564
 565 static int
 566 mlxsw_emad_process_status_skb(struct sk_buff *skb,
 567                               enum mlxsw_emad_op_tlv_status *p_status)
 568 {
 569         return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
 570 }
 571
 572 struct mlxsw_reg_trans {
 573         struct list_head list;
 574         struct list_head bulk_list;
 575         struct mlxsw_core *core;
 576         struct sk_buff *tx_skb;
 577         struct mlxsw_tx_info tx_info;
 578         struct delayed_work timeout_dw;
 579         unsigned int retries;
 580         u64 tid;
 581         struct completion completion;
 582         atomic_t active;
 583         mlxsw_reg_trans_cb_t *cb;
 584         unsigned long cb_priv;
 585         const struct mlxsw_reg_info *reg;
 586         enum mlxsw_core_reg_access_type type;
 587         int err;
 588         char *emad_err_string;
 589         enum mlxsw_emad_op_tlv_status emad_status;
 590         struct rcu_head rcu;
 591 };
 592
 593 static void mlxsw_emad_process_string_tlv(const struct sk_buff *skb,
 594                                           struct mlxsw_reg_trans *trans)
 595 {
 596         char *string_tlv;
 597         char *string;
 598
 599         string_tlv = mlxsw_emad_string_tlv(skb);
 600         if (!string_tlv)
 601                 return;
 602
 603         trans->emad_err_string = kzalloc(MLXSW_EMAD_STRING_TLV_STRING_LEN,
 604                                          GFP_ATOMIC);
 605         if (!trans->emad_err_string)
 606                 return;
 607
 608         string = mlxsw_emad_string_tlv_string_data(string_tlv);
 609         strlcpy(trans->emad_err_string, string,
 610                 MLXSW_EMAD_STRING_TLV_STRING_LEN);
 611 }
 612
 613 #define MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS   3000
 614 #define MLXSW_EMAD_TIMEOUT_MS                   200
 615
 616 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
 617 {
 618         unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
 619
 620         if (trans->core->fw_flash_in_progress)
 621                 timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS);
 622
 623         queue_delayed_work(trans->core->emad_wq, &trans->timeout_dw,
 624                            timeout << trans->retries);
 625 }
 626
 627 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
 628                                struct mlxsw_reg_trans *trans)
 629 {
 630         struct sk_buff *skb;
 631         int err;
 632
 633         skb = skb_copy(trans->tx_skb, GFP_KERNEL);
 634         if (!skb)
 635                 return -ENOMEM;
 636
 637         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
 638                             skb->data + mlxsw_core->driver->txhdr_len,
 639                             skb->len - mlxsw_core->driver->txhdr_len);
 640
 641         atomic_set(&trans->active, 1);
 642         err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
 643         if (err) {
 644                 dev_kfree_skb(skb);
 645                 return err;
 646         }
 647         mlxsw_emad_trans_timeout_schedule(trans);
 648         return 0;
 649 }
 650
 651 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
 652 {
 653         struct mlxsw_core *mlxsw_core = trans->core;
 654
 655         dev_kfree_skb(trans->tx_skb);
 656         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 657         list_del_rcu(&trans->list);
 658         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 659         trans->err = err;
 660         complete(&trans->completion);
 661 }
 662
 663 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
 664                                       struct mlxsw_reg_trans *trans)
 665 {
 666         int err;
 667
 668         if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
 669                 trans->retries++;
 670                 err = mlxsw_emad_transmit(trans->core, trans);
 671                 if (err == 0)
 672                         return;
 673
 674                 if (!atomic_dec_and_test(&trans->active))
 675                         return;
 676         } else {
 677                 err = -EIO;
 678         }
 679         mlxsw_emad_trans_finish(trans, err);
 680 }
 681
 682 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
 683 {
 684         struct mlxsw_reg_trans *trans = container_of(work,
 685                                                      struct mlxsw_reg_trans,
 686                                                      timeout_dw.work);
 687
 688         if (!atomic_dec_and_test(&trans->active))
 689                 return;
 690
 691         mlxsw_emad_transmit_retry(trans->core, trans);
 692 }
 693
 694 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
 695                                         struct mlxsw_reg_trans *trans,
 696                                         struct sk_buff *skb)
 697 {
 698         int err;
 699
 700         if (!atomic_dec_and_test(&trans->active))
 701                 return;
 702
 703         err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
 704         if (err == -EAGAIN) {
 705                 mlxsw_emad_transmit_retry(mlxsw_core, trans);
 706         } else {
 707                 if (err == 0) {
 708                         char *reg_tlv = mlxsw_emad_reg_tlv(skb);
 709
 710                         if (trans->cb)
 711                                 trans->cb(mlxsw_core,
 712                                           mlxsw_emad_reg_payload(reg_tlv),
 713                                           trans->reg->len, trans->cb_priv);
 714                 } else {
 715                         mlxsw_emad_process_string_tlv(skb, trans);
 716                 }
 717                 mlxsw_emad_trans_finish(trans, err);
 718         }
 719 }
 720
 721 /* called with rcu read lock held */
 722 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port,
 723                                         void *priv)
 724 {
 725         struct mlxsw_core *mlxsw_core = priv;
 726         struct mlxsw_reg_trans *trans;
 727
 728         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
 729                             skb->data, skb->len);
 730
 731         mlxsw_emad_tlv_parse(skb);
 732
 733         if (!mlxsw_emad_is_resp(skb))
 734                 goto free_skb;
 735
 736         list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
 737                 if (mlxsw_emad_get_tid(skb) == trans->tid) {
 738                         mlxsw_emad_process_response(mlxsw_core, trans, skb);
 739                         break;
 740                 }
 741         }
 742
 743 free_skb:
 744         dev_kfree_skb(skb);
 745 }
 746
 747 static const struct mlxsw_listener mlxsw_emad_rx_listener =
 748         MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
 749                   EMAD, DISCARD);
 750
 751 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
 752 {
 753         struct workqueue_struct *emad_wq;
 754         u64 tid;
 755         int err;
 756
 757         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 758                 return 0;
 759
 760         emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0);
 761         if (!emad_wq)
 762                 return -ENOMEM;
 763         mlxsw_core->emad_wq = emad_wq;
 764
 765         /* Set the upper 32 bits of the transaction ID field to a random
 766          * number. This allows us to discard EMADs addressed to other
 767          * devices.
 768          */
 769         get_random_bytes(&tid, 4);
 770         tid <<= 32;
 771         atomic64_set(&mlxsw_core->emad.tid, tid);
 772
 773         INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
 774         spin_lock_init(&mlxsw_core->emad.trans_list_lock);
 775
 776         err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
 777                                        mlxsw_core);
 778         if (err)
 779                 goto err_trap_register;
 780
 781         err = mlxsw_core->driver->basic_trap_groups_set(mlxsw_core);
 782         if (err)
 783                 goto err_emad_trap_set;
 784         mlxsw_core->emad.use_emad = true;
 785
 786         return 0;
 787
 788 err_emad_trap_set:
 789         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 790                                    mlxsw_core);
 791 err_trap_register:
 792         destroy_workqueue(mlxsw_core->emad_wq);
 793         return err;
 794 }
 795
 796 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
 797 {
 798
 799         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 800                 return;
 801
 802         mlxsw_core->emad.use_emad = false;
 803         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 804                                    mlxsw_core);
 805         destroy_workqueue(mlxsw_core->emad_wq);
 806 }
 807
 808 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
 809                                         u16 reg_len, bool enable_string_tlv)
 810 {
 811         struct sk_buff *skb;
 812         u16 emad_len;
 813
 814         emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
 815                     (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
 816                     sizeof(u32) + mlxsw_core->driver->txhdr_len);
 817         if (enable_string_tlv)
 818                 emad_len += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
 819         if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
 820                 return NULL;
 821
 822         skb = netdev_alloc_skb(NULL, emad_len);
 823         if (!skb)
 824                 return NULL;
 825         memset(skb->data, 0, emad_len);
 826         skb_reserve(skb, emad_len);
 827
 828         return skb;
 829 }
 830
 831 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
 832                                  const struct mlxsw_reg_info *reg,
 833                                  char *payload,
 834                                  enum mlxsw_core_reg_access_type type,
 835                                  struct mlxsw_reg_trans *trans,
 836                                  struct list_head *bulk_list,
 837                                  mlxsw_reg_trans_cb_t *cb,
 838                                  unsigned long cb_priv, u64 tid)
 839 {
 840         bool enable_string_tlv;
 841         struct sk_buff *skb;
 842         int err;
 843
 844         dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
 845                 tid, reg->id, mlxsw_reg_id_str(reg->id),
 846                 mlxsw_core_reg_access_type_str(type));
 847
 848         /* Since this can be changed during emad_reg_access, read it once and
 849          * use the value all the way.
 850          */
 851         enable_string_tlv = mlxsw_core->emad.enable_string_tlv;
 852
 853         skb = mlxsw_emad_alloc(mlxsw_core, reg->len, enable_string_tlv);
 854         if (!skb)
 855                 return -ENOMEM;
 856
 857         list_add_tail(&trans->bulk_list, bulk_list);
 858         trans->core = mlxsw_core;
 859         trans->tx_skb = skb;
 860         trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
 861         trans->tx_info.is_emad = true;
 862         INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
 863         trans->tid = tid;
 864         init_completion(&trans->completion);
 865         trans->cb = cb;
 866         trans->cb_priv = cb_priv;
 867         trans->reg = reg;
 868         trans->type = type;
 869
 870         mlxsw_emad_construct(skb, reg, payload, type, trans->tid,
 871                              enable_string_tlv);
 872         mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
 873
 874         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 875         list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
 876         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 877         err = mlxsw_emad_transmit(mlxsw_core, trans);
 878         if (err)
 879                 goto err_out;
 880         return 0;
 881
 882 err_out:
 883         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 884         list_del_rcu(&trans->list);
 885         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 886         list_del(&trans->bulk_list);
 887         dev_kfree_skb(trans->tx_skb);
 888         return err;
 889 }
 890
 891 /*****************
 892  * Core functions
 893  *****************/
 894
 895 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
 896 {
 897         spin_lock(&mlxsw_core_driver_list_lock);
 898         list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
 899         spin_unlock(&mlxsw_core_driver_list_lock);
 900         return 0;
 901 }
 902 EXPORT_SYMBOL(mlxsw_core_driver_register);
 903
 904 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
 905 {
 906         spin_lock(&mlxsw_core_driver_list_lock);
 907         list_del(&mlxsw_driver->list);
 908         spin_unlock(&mlxsw_core_driver_list_lock);
 909 }
 910 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
 911
 912 static struct mlxsw_driver *__driver_find(const char *kind)
 913 {
 914         struct mlxsw_driver *mlxsw_driver;
 915
 916         list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
 917                 if (strcmp(mlxsw_driver->kind, kind) == 0)
 918                         return mlxsw_driver;
 919         }
 920         return NULL;
 921 }
 922
 923 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
 924 {
 925         struct mlxsw_driver *mlxsw_driver;
 926
 927         spin_lock(&mlxsw_core_driver_list_lock);
 928         mlxsw_driver = __driver_find(kind);
 929         spin_unlock(&mlxsw_core_driver_list_lock);
 930         return mlxsw_driver;
 931 }
 932
 933 struct mlxsw_core_fw_info {
 934         struct mlxfw_dev mlxfw_dev;
 935         struct mlxsw_core *mlxsw_core;
 936 };
 937
 938 static int mlxsw_core_fw_component_query(struct mlxfw_dev *mlxfw_dev,
 939                                          u16 component_index, u32 *p_max_size,
 940                                          u8 *p_align_bits, u16 *p_max_write_size)
 941 {
 942         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
 943                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
 944         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
 945         char mcqi_pl[MLXSW_REG_MCQI_LEN];
 946         int err;
 947
 948         mlxsw_reg_mcqi_pack(mcqi_pl, component_index);
 949         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcqi), mcqi_pl);
 950         if (err)
 951                 return err;
 952         mlxsw_reg_mcqi_unpack(mcqi_pl, p_max_size, p_align_bits, p_max_write_size);
 953
 954         *p_align_bits = max_t(u8, *p_align_bits, 2);
 955         *p_max_write_size = min_t(u16, *p_max_write_size, MLXSW_REG_MCDA_MAX_DATA_LEN);
 956         return 0;
 957 }
 958
 959 static int mlxsw_core_fw_fsm_lock(struct mlxfw_dev *mlxfw_dev, u32 *fwhandle)
 960 {
 961         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
 962                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
 963         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
 964         char mcc_pl[MLXSW_REG_MCC_LEN];
 965         u8 control_state;
 966         int err;
 967
 968         mlxsw_reg_mcc_pack(mcc_pl, 0, 0, 0, 0);
 969         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
 970         if (err)
 971                 return err;
 972
 973         mlxsw_reg_mcc_unpack(mcc_pl, fwhandle, NULL, &control_state);
 974         if (control_state != MLXFW_FSM_STATE_IDLE)
 975                 return -EBUSY;
 976
 977         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_LOCK_UPDATE_HANDLE, 0, *fwhandle, 0);
 978         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
 979 }
 980
 981 static int mlxsw_core_fw_fsm_component_update(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
 982                                               u16 component_index, u32 component_size)
 983 {
 984         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
 985                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
 986         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
 987         char mcc_pl[MLXSW_REG_MCC_LEN];
 988
 989         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_UPDATE_COMPONENT,
 990                            component_index, fwhandle, component_size);
 991         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
 992 }
 993
 994 static int mlxsw_core_fw_fsm_block_download(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
 995                                             u8 *data, u16 size, u32 offset)
 996 {
 997         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
 998                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
 999         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1000         char mcda_pl[MLXSW_REG_MCDA_LEN];
1001
1002         mlxsw_reg_mcda_pack(mcda_pl, fwhandle, offset, size, data);
1003         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcda), mcda_pl);
1004 }
1005
1006 static int mlxsw_core_fw_fsm_component_verify(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1007                                               u16 component_index)
1008 {
1009         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1010                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1011         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1012         char mcc_pl[MLXSW_REG_MCC_LEN];
1013
1014         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_VERIFY_COMPONENT,
1015                            component_index, fwhandle, 0);
1016         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1017 }
1018
1019 static int mlxsw_core_fw_fsm_activate(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1020 {
1021         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1022                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1023         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1024         char mcc_pl[MLXSW_REG_MCC_LEN];
1025
1026         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_ACTIVATE, 0, fwhandle, 0);
1027         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1028 }
1029
1030 static int mlxsw_core_fw_fsm_query_state(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1031                                          enum mlxfw_fsm_state *fsm_state,
1032                                          enum mlxfw_fsm_state_err *fsm_state_err)
1033 {
1034         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1035                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1036         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1037         char mcc_pl[MLXSW_REG_MCC_LEN];
1038         u8 control_state;
1039         u8 error_code;
1040         int err;
1041
1042         mlxsw_reg_mcc_pack(mcc_pl, 0, 0, fwhandle, 0);
1043         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1044         if (err)
1045                 return err;
1046
1047         mlxsw_reg_mcc_unpack(mcc_pl, NULL, &error_code, &control_state);
1048         *fsm_state = control_state;
1049         *fsm_state_err = min_t(enum mlxfw_fsm_state_err, error_code, MLXFW_FSM_STATE_ERR_MAX);
1050         return 0;
1051 }
1052
1053 static void mlxsw_core_fw_fsm_cancel(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1054 {
1055         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1056                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1057         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1058         char mcc_pl[MLXSW_REG_MCC_LEN];
1059
1060         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_CANCEL, 0, fwhandle, 0);
1061         mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1062 }
1063
1064 static void mlxsw_core_fw_fsm_release(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1065 {
1066         struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1067                 container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1068         struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1069         char mcc_pl[MLXSW_REG_MCC_LEN];
1070
1071         mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_RELEASE_UPDATE_HANDLE, 0, fwhandle, 0);
1072         mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1073 }
1074
1075 static const struct mlxfw_dev_ops mlxsw_core_fw_mlxsw_dev_ops = {
1076         .component_query        = mlxsw_core_fw_component_query,
1077         .fsm_lock               = mlxsw_core_fw_fsm_lock,
1078         .fsm_component_update   = mlxsw_core_fw_fsm_component_update,
1079         .fsm_block_download     = mlxsw_core_fw_fsm_block_download,
1080         .fsm_component_verify   = mlxsw_core_fw_fsm_component_verify,
1081         .fsm_activate           = mlxsw_core_fw_fsm_activate,
1082         .fsm_query_state        = mlxsw_core_fw_fsm_query_state,
1083         .fsm_cancel             = mlxsw_core_fw_fsm_cancel,
1084         .fsm_release            = mlxsw_core_fw_fsm_release,
1085 };
1086
1087 static int mlxsw_core_fw_flash(struct mlxsw_core *mlxsw_core, const struct firmware *firmware,
1088                                struct netlink_ext_ack *extack)
1089 {
1090         struct mlxsw_core_fw_info mlxsw_core_fw_info = {
1091                 .mlxfw_dev = {
1092                         .ops = &mlxsw_core_fw_mlxsw_dev_ops,
1093                         .psid = mlxsw_core->bus_info->psid,
1094                         .psid_size = strlen(mlxsw_core->bus_info->psid),
1095                         .devlink = priv_to_devlink(mlxsw_core),
1096                 },
1097                 .mlxsw_core = mlxsw_core
1098         };
1099         int err;
1100
1101         mlxsw_core->fw_flash_in_progress = true;
1102         err = mlxfw_firmware_flash(&mlxsw_core_fw_info.mlxfw_dev, firmware, extack);
1103         mlxsw_core->fw_flash_in_progress = false;
1104
1105         return err;
1106 }
1107
1108 static int mlxsw_core_fw_rev_validate(struct mlxsw_core *mlxsw_core,
1109                                       const struct mlxsw_bus_info *mlxsw_bus_info,
1110                                       const struct mlxsw_fw_rev *req_rev,
1111                                       const char *filename)
1112 {
1113         const struct mlxsw_fw_rev *rev = &mlxsw_bus_info->fw_rev;
1114         union devlink_param_value value;
1115         const struct firmware *firmware;
1116         int err;
1117
1118         /* Don't check if driver does not require it */
1119         if (!req_rev || !filename)
1120                 return 0;
1121
1122         /* Don't check if devlink 'fw_load_policy' param is 'flash' */
1123         err = devlink_param_driverinit_value_get(priv_to_devlink(mlxsw_core),
1124                                                  DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
1125                                                  &value);
1126         if (err)
1127                 return err;
1128         if (value.vu8 == DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_FLASH)
1129                 return 0;
1130
1131         /* Validate driver & FW are compatible */
1132         if (rev->major != req_rev->major) {
1133                 WARN(1, "Mismatch in major FW version [%d:%d] is never expected; Please contact support\n",
1134                      rev->major, req_rev->major);
1135                 return -EINVAL;
1136         }
1137         if (mlxsw_core_fw_rev_minor_subminor_validate(rev, req_rev))
1138                 return 0;
1139
1140         dev_err(mlxsw_bus_info->dev, "The firmware version %d.%d.%d is incompatible with the driver (required >= %d.%d.%d)\n",
1141                 rev->major, rev->minor, rev->subminor, req_rev->major,
1142                 req_rev->minor, req_rev->subminor);
1143         dev_info(mlxsw_bus_info->dev, "Flashing firmware using file %s\n", filename);
1144
1145         err = request_firmware_direct(&firmware, filename, mlxsw_bus_info->dev);
1146         if (err) {
1147                 dev_err(mlxsw_bus_info->dev, "Could not request firmware file %s\n", filename);
1148                 return err;
1149         }
1150
1151         err = mlxsw_core_fw_flash(mlxsw_core, firmware, NULL);
1152         release_firmware(firmware);
1153         if (err)
1154                 dev_err(mlxsw_bus_info->dev, "Could not upgrade firmware\n");
1155
1156         /* On FW flash success, tell the caller FW reset is needed
1157          * if current FW supports it.
1158          */
1159         if (rev->minor >= req_rev->can_reset_minor)
1160                 return err ? err : -EAGAIN;
1161         else
1162                 return 0;
1163 }
1164
1165 static int mlxsw_core_fw_flash_update(struct mlxsw_core *mlxsw_core,
1166                                       struct devlink_flash_update_params *params,
1167                                       struct netlink_ext_ack *extack)
1168 {
1169         return mlxsw_core_fw_flash(mlxsw_core, params->fw, extack);
1170 }
1171
1172 static int mlxsw_core_devlink_param_fw_load_policy_validate(struct devlink *devlink, u32 id,
1173                                                             union devlink_param_value val,
1174                                                             struct netlink_ext_ack *extack)
1175 {
1176         if (val.vu8 != DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DRIVER &&
1177             val.vu8 != DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_FLASH) {
1178                 NL_SET_ERR_MSG_MOD(extack, "'fw_load_policy' must be 'driver' or 'flash'");
1179                 return -EINVAL;
1180         }
1181
1182         return 0;
1183 }
1184
1185 static const struct devlink_param mlxsw_core_fw_devlink_params[] = {
1186         DEVLINK_PARAM_GENERIC(FW_LOAD_POLICY, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
1187                               mlxsw_core_devlink_param_fw_load_policy_validate),
1188 };
1189
1190 static int mlxsw_core_fw_params_register(struct mlxsw_core *mlxsw_core)
1191 {
1192         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1193         union devlink_param_value value;
1194         int err;
1195
1196         err = devlink_params_register(devlink, mlxsw_core_fw_devlink_params,
1197                                       ARRAY_SIZE(mlxsw_core_fw_devlink_params));
1198         if (err)
1199                 return err;
1200
1201         value.vu8 = DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DRIVER;
1202         devlink_param_driverinit_value_set(devlink, DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY, value);
1203         return 0;
1204 }
1205
1206 static void mlxsw_core_fw_params_unregister(struct mlxsw_core *mlxsw_core)
1207 {
1208         devlink_params_unregister(priv_to_devlink(mlxsw_core), mlxsw_core_fw_devlink_params,
1209                                   ARRAY_SIZE(mlxsw_core_fw_devlink_params));
1210 }
1211
1212 static int mlxsw_devlink_port_split(struct devlink *devlink,
1213                                     unsigned int port_index,
1214                                     unsigned int count,
1215                                     struct netlink_ext_ack *extack)
1216 {
1217         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1218
1219         if (port_index >= mlxsw_core->max_ports) {
1220                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
1221                 return -EINVAL;
1222         }
1223         if (!mlxsw_core->driver->port_split)
1224                 return -EOPNOTSUPP;
1225         return mlxsw_core->driver->port_split(mlxsw_core, port_index, count,
1226                                               extack);
1227 }
1228
1229 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
1230                                       unsigned int port_index,
1231                                       struct netlink_ext_ack *extack)
1232 {
1233         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1234
1235         if (port_index >= mlxsw_core->max_ports) {
1236                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
1237                 return -EINVAL;
1238         }
1239         if (!mlxsw_core->driver->port_unsplit)
1240                 return -EOPNOTSUPP;
1241         return mlxsw_core->driver->port_unsplit(mlxsw_core, port_index,
1242                                                 extack);
1243 }
1244
1245 static int
1246 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
1247                           unsigned int sb_index, u16 pool_index,
1248                           struct devlink_sb_pool_info *pool_info)
1249 {
1250         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1251         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1252
1253         if (!mlxsw_driver->sb_pool_get)
1254                 return -EOPNOTSUPP;
1255         return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
1256                                          pool_index, pool_info);
1257 }
1258
1259 static int
1260 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
1261                           unsigned int sb_index, u16 pool_index, u32 size,
1262                           enum devlink_sb_threshold_type threshold_type,
1263                           struct netlink_ext_ack *extack)
1264 {
1265         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1266         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1267
1268         if (!mlxsw_driver->sb_pool_set)
1269                 return -EOPNOTSUPP;
1270         return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
1271                                          pool_index, size, threshold_type,
1272                                          extack);
1273 }
1274
1275 static void *__dl_port(struct devlink_port *devlink_port)
1276 {
1277         return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
1278 }
1279
1280 static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
1281                                        enum devlink_port_type port_type)
1282 {
1283         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1284         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1285         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1286
1287         if (!mlxsw_driver->port_type_set)
1288                 return -EOPNOTSUPP;
1289
1290         return mlxsw_driver->port_type_set(mlxsw_core,
1291                                            mlxsw_core_port->local_port,
1292                                            port_type);
1293 }
1294
1295 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
1296                                           unsigned int sb_index, u16 pool_index,
1297                                           u32 *p_threshold)
1298 {
1299         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1300         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1301         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1302
1303         if (!mlxsw_driver->sb_port_pool_get ||
1304             !mlxsw_core_port_check(mlxsw_core_port))
1305                 return -EOPNOTSUPP;
1306         return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
1307                                               pool_index, p_threshold);
1308 }
1309
1310 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
1311                                           unsigned int sb_index, u16 pool_index,
1312                                           u32 threshold,
1313                                           struct netlink_ext_ack *extack)
1314 {
1315         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1316         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1317         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1318
1319         if (!mlxsw_driver->sb_port_pool_set ||
1320             !mlxsw_core_port_check(mlxsw_core_port))
1321                 return -EOPNOTSUPP;
1322         return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
1323                                               pool_index, threshold, extack);
1324 }
1325
1326 static int
1327 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
1328                                   unsigned int sb_index, u16 tc_index,
1329                                   enum devlink_sb_pool_type pool_type,
1330                                   u16 *p_pool_index, u32 *p_threshold)
1331 {
1332         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1333         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1334         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1335
1336         if (!mlxsw_driver->sb_tc_pool_bind_get ||
1337             !mlxsw_core_port_check(mlxsw_core_port))
1338                 return -EOPNOTSUPP;
1339         return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
1340                                                  tc_index, pool_type,
1341                                                  p_pool_index, p_threshold);
1342 }
1343
1344 static int
1345 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1346                                   unsigned int sb_index, u16 tc_index,
1347                                   enum devlink_sb_pool_type pool_type,
1348                                   u16 pool_index, u32 threshold,
1349                                   struct netlink_ext_ack *extack)
1350 {
1351         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1352         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1353         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1354
1355         if (!mlxsw_driver->sb_tc_pool_bind_set ||
1356             !mlxsw_core_port_check(mlxsw_core_port))
1357                 return -EOPNOTSUPP;
1358         return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
1359                                                  tc_index, pool_type,
1360                                                  pool_index, threshold, extack);
1361 }
1362
1363 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
1364                                          unsigned int sb_index)
1365 {
1366         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1367         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1368
1369         if (!mlxsw_driver->sb_occ_snapshot)
1370                 return -EOPNOTSUPP;
1371         return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
1372 }
1373
1374 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
1375                                           unsigned int sb_index)
1376 {
1377         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1378         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1379
1380         if (!mlxsw_driver->sb_occ_max_clear)
1381                 return -EOPNOTSUPP;
1382         return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
1383 }
1384
1385 static int
1386 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
1387                                    unsigned int sb_index, u16 pool_index,
1388                                    u32 *p_cur, u32 *p_max)
1389 {
1390         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1391         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1392         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1393
1394         if (!mlxsw_driver->sb_occ_port_pool_get ||
1395             !mlxsw_core_port_check(mlxsw_core_port))
1396                 return -EOPNOTSUPP;
1397         return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
1398                                                   pool_index, p_cur, p_max);
1399 }
1400
1401 static int
1402 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
1403                                       unsigned int sb_index, u16 tc_index,
1404                                       enum devlink_sb_pool_type pool_type,
1405                                       u32 *p_cur, u32 *p_max)
1406 {
1407         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1408         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1409         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1410
1411         if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
1412             !mlxsw_core_port_check(mlxsw_core_port))
1413                 return -EOPNOTSUPP;
1414         return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
1415                                                      sb_index, tc_index,
1416                                                      pool_type, p_cur, p_max);
1417 }
1418
1419 static int
1420 mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
1421                        struct netlink_ext_ack *extack)
1422 {
1423         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1424         char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE];
1425         u32 hw_rev, fw_major, fw_minor, fw_sub_minor;
1426         char mgir_pl[MLXSW_REG_MGIR_LEN];
1427         char buf[32];
1428         int err;
1429
1430         err = devlink_info_driver_name_put(req,
1431                                            mlxsw_core->bus_info->device_kind);
1432         if (err)
1433                 return err;
1434
1435         mlxsw_reg_mgir_pack(mgir_pl);
1436         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
1437         if (err)
1438                 return err;
1439         mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major,
1440                               &fw_minor, &fw_sub_minor);
1441
1442         sprintf(buf, "%X", hw_rev);
1443         err = devlink_info_version_fixed_put(req, "hw.revision", buf);
1444         if (err)
1445                 return err;
1446
1447         err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid);
1448         if (err)
1449                 return err;
1450
1451         sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor);
1452         err = devlink_info_version_running_put(req, "fw.version", buf);
1453         if (err)
1454                 return err;
1455
1456         return 0;
1457 }
1458
1459 static int
1460 mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink,
1461                                           bool netns_change, enum devlink_reload_action action,
1462                                           enum devlink_reload_limit limit,
1463                                           struct netlink_ext_ack *extack)
1464 {
1465         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1466
1467         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
1468                 return -EOPNOTSUPP;
1469
1470         mlxsw_core_bus_device_unregister(mlxsw_core, true);
1471         return 0;
1472 }
1473
1474 static int
1475 mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink, enum devlink_reload_action action,
1476                                         enum devlink_reload_limit limit, u32 *actions_performed,
1477                                         struct netlink_ext_ack *extack)
1478 {
1479         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1480
1481         *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
1482                              BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE);
1483         return mlxsw_core_bus_device_register(mlxsw_core->bus_info,
1484                                               mlxsw_core->bus,
1485                                               mlxsw_core->bus_priv, true,
1486                                               devlink, extack);
1487 }
1488
1489 static int mlxsw_devlink_flash_update(struct devlink *devlink,
1490                                       struct devlink_flash_update_params *params,
1491                                       struct netlink_ext_ack *extack)
1492 {
1493         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1494
1495         return mlxsw_core_fw_flash_update(mlxsw_core, params, extack);
1496 }
1497
1498 static int mlxsw_devlink_trap_init(struct devlink *devlink,
1499                                    const struct devlink_trap *trap,
1500                                    void *trap_ctx)
1501 {
1502         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1503         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1504
1505         if (!mlxsw_driver->trap_init)
1506                 return -EOPNOTSUPP;
1507         return mlxsw_driver->trap_init(mlxsw_core, trap, trap_ctx);
1508 }
1509
1510 static void mlxsw_devlink_trap_fini(struct devlink *devlink,
1511                                     const struct devlink_trap *trap,
1512                                     void *trap_ctx)
1513 {
1514         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1515         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1516
1517         if (!mlxsw_driver->trap_fini)
1518                 return;
1519         mlxsw_driver->trap_fini(mlxsw_core, trap, trap_ctx);
1520 }
1521
1522 static int mlxsw_devlink_trap_action_set(struct devlink *devlink,
1523                                          const struct devlink_trap *trap,
1524                                          enum devlink_trap_action action,
1525                                          struct netlink_ext_ack *extack)
1526 {
1527         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1528         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1529
1530         if (!mlxsw_driver->trap_action_set)
1531                 return -EOPNOTSUPP;
1532         return mlxsw_driver->trap_action_set(mlxsw_core, trap, action, extack);
1533 }
1534
1535 static int
1536 mlxsw_devlink_trap_group_init(struct devlink *devlink,
1537                               const struct devlink_trap_group *group)
1538 {
1539         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1540         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1541
1542         if (!mlxsw_driver->trap_group_init)
1543                 return -EOPNOTSUPP;
1544         return mlxsw_driver->trap_group_init(mlxsw_core, group);
1545 }
1546
1547 static int
1548 mlxsw_devlink_trap_group_set(struct devlink *devlink,
1549                              const struct devlink_trap_group *group,
1550                              const struct devlink_trap_policer *policer,
1551                              struct netlink_ext_ack *extack)
1552 {
1553         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1554         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1555
1556         if (!mlxsw_driver->trap_group_set)
1557                 return -EOPNOTSUPP;
1558         return mlxsw_driver->trap_group_set(mlxsw_core, group, policer, extack);
1559 }
1560
1561 static int
1562 mlxsw_devlink_trap_policer_init(struct devlink *devlink,
1563                                 const struct devlink_trap_policer *policer)
1564 {
1565         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1566         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1567
1568         if (!mlxsw_driver->trap_policer_init)
1569                 return -EOPNOTSUPP;
1570         return mlxsw_driver->trap_policer_init(mlxsw_core, policer);
1571 }
1572
1573 static void
1574 mlxsw_devlink_trap_policer_fini(struct devlink *devlink,
1575                                 const struct devlink_trap_policer *policer)
1576 {
1577         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1578         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1579
1580         if (!mlxsw_driver->trap_policer_fini)
1581                 return;
1582         mlxsw_driver->trap_policer_fini(mlxsw_core, policer);
1583 }
1584
1585 static int
1586 mlxsw_devlink_trap_policer_set(struct devlink *devlink,
1587                                const struct devlink_trap_policer *policer,
1588                                u64 rate, u64 burst,
1589                                struct netlink_ext_ack *extack)
1590 {
1591         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1592         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1593
1594         if (!mlxsw_driver->trap_policer_set)
1595                 return -EOPNOTSUPP;
1596         return mlxsw_driver->trap_policer_set(mlxsw_core, policer, rate, burst,
1597                                               extack);
1598 }
1599
1600 static int
1601 mlxsw_devlink_trap_policer_counter_get(struct devlink *devlink,
1602                                        const struct devlink_trap_policer *policer,
1603                                        u64 *p_drops)
1604 {
1605         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1606         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1607
1608         if (!mlxsw_driver->trap_policer_counter_get)
1609                 return -EOPNOTSUPP;
1610         return mlxsw_driver->trap_policer_counter_get(mlxsw_core, policer,
1611                                                       p_drops);
1612 }
1613
1614 static const struct devlink_ops mlxsw_devlink_ops = {
1615         .reload_actions         = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
1616                                   BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
1617         .reload_down            = mlxsw_devlink_core_bus_device_reload_down,
1618         .reload_up              = mlxsw_devlink_core_bus_device_reload_up,
1619         .port_type_set                  = mlxsw_devlink_port_type_set,
1620         .port_split                     = mlxsw_devlink_port_split,
1621         .port_unsplit                   = mlxsw_devlink_port_unsplit,
1622         .sb_pool_get                    = mlxsw_devlink_sb_pool_get,
1623         .sb_pool_set                    = mlxsw_devlink_sb_pool_set,
1624         .sb_port_pool_get               = mlxsw_devlink_sb_port_pool_get,
1625         .sb_port_pool_set               = mlxsw_devlink_sb_port_pool_set,
1626         .sb_tc_pool_bind_get            = mlxsw_devlink_sb_tc_pool_bind_get,
1627         .sb_tc_pool_bind_set            = mlxsw_devlink_sb_tc_pool_bind_set,
1628         .sb_occ_snapshot                = mlxsw_devlink_sb_occ_snapshot,
1629         .sb_occ_max_clear               = mlxsw_devlink_sb_occ_max_clear,
1630         .sb_occ_port_pool_get           = mlxsw_devlink_sb_occ_port_pool_get,
1631         .sb_occ_tc_port_bind_get        = mlxsw_devlink_sb_occ_tc_port_bind_get,
1632         .info_get                       = mlxsw_devlink_info_get,
1633         .flash_update                   = mlxsw_devlink_flash_update,
1634         .trap_init                      = mlxsw_devlink_trap_init,
1635         .trap_fini                      = mlxsw_devlink_trap_fini,
1636         .trap_action_set                = mlxsw_devlink_trap_action_set,
1637         .trap_group_init                = mlxsw_devlink_trap_group_init,
1638         .trap_group_set                 = mlxsw_devlink_trap_group_set,
1639         .trap_policer_init              = mlxsw_devlink_trap_policer_init,
1640         .trap_policer_fini              = mlxsw_devlink_trap_policer_fini,
1641         .trap_policer_set               = mlxsw_devlink_trap_policer_set,
1642         .trap_policer_counter_get       = mlxsw_devlink_trap_policer_counter_get,
1643 };
1644
1645 static int mlxsw_core_params_register(struct mlxsw_core *mlxsw_core)
1646 {
1647         int err;
1648
1649         err = mlxsw_core_fw_params_register(mlxsw_core);
1650         if (err)
1651                 return err;
1652
1653         if (mlxsw_core->driver->params_register) {
1654                 err = mlxsw_core->driver->params_register(mlxsw_core);
1655                 if (err)
1656                         goto err_params_register;
1657         }
1658         return 0;
1659
1660 err_params_register:
1661         mlxsw_core_fw_params_unregister(mlxsw_core);
1662         return err;
1663 }
1664
1665 static void mlxsw_core_params_unregister(struct mlxsw_core *mlxsw_core)
1666 {
1667         mlxsw_core_fw_params_unregister(mlxsw_core);
1668         if (mlxsw_core->driver->params_register)
1669                 mlxsw_core->driver->params_unregister(mlxsw_core);
1670 }
1671
1672 struct mlxsw_core_health_event {
1673         struct mlxsw_core *mlxsw_core;
1674         char mfde_pl[MLXSW_REG_MFDE_LEN];
1675         struct work_struct work;
1676 };
1677
1678 static void mlxsw_core_health_event_work(struct work_struct *work)
1679 {
1680         struct mlxsw_core_health_event *event;
1681         struct mlxsw_core *mlxsw_core;
1682
1683         event = container_of(work, struct mlxsw_core_health_event, work);
1684         mlxsw_core = event->mlxsw_core;
1685         devlink_health_report(mlxsw_core->health.fw_fatal, "FW fatal event occurred",
1686                               event->mfde_pl);
1687         kfree(event);
1688 }
1689
1690 static void mlxsw_core_health_listener_func(const struct mlxsw_reg_info *reg,
1691                                             char *mfde_pl, void *priv)
1692 {
1693         struct mlxsw_core_health_event *event;
1694         struct mlxsw_core *mlxsw_core = priv;
1695
1696         event = kmalloc(sizeof(*event), GFP_ATOMIC);
1697         if (!event)
1698                 return;
1699         event->mlxsw_core = mlxsw_core;
1700         memcpy(event->mfde_pl, mfde_pl, sizeof(event->mfde_pl));
1701         INIT_WORK(&event->work, mlxsw_core_health_event_work);
1702         mlxsw_core_schedule_work(&event->work);
1703 }
1704
1705 static const struct mlxsw_listener mlxsw_core_health_listener =
1706         MLXSW_EVENTL(mlxsw_core_health_listener_func, MFDE, MFDE);
1707
1708 static int mlxsw_core_health_fw_fatal_dump(struct devlink_health_reporter *reporter,
1709                                            struct devlink_fmsg *fmsg, void *priv_ctx,
1710                                            struct netlink_ext_ack *extack)
1711 {
1712         char *mfde_pl = priv_ctx;
1713         char *val_str;
1714         u8 event_id;
1715         u32 val;
1716         int err;
1717
1718         if (!priv_ctx)
1719                 /* User-triggered dumps are not possible */
1720                 return -EOPNOTSUPP;
1721
1722         val = mlxsw_reg_mfde_irisc_id_get(mfde_pl);
1723         err = devlink_fmsg_u8_pair_put(fmsg, "irisc_id", val);
1724         if (err)
1725                 return err;
1726         err = devlink_fmsg_arr_pair_nest_start(fmsg, "event");
1727         if (err)
1728                 return err;
1729
1730         event_id = mlxsw_reg_mfde_event_id_get(mfde_pl);
1731         err = devlink_fmsg_u32_pair_put(fmsg, "id", event_id);
1732         if (err)
1733                 return err;
1734         switch (event_id) {
1735         case MLXSW_REG_MFDE_EVENT_ID_CRSPACE_TO:
1736                 val_str = "CR space timeout";
1737                 break;
1738         case MLXSW_REG_MFDE_EVENT_ID_KVD_IM_STOP:
1739                 val_str = "KVD insertion machine stopped";
1740                 break;
1741         default:
1742                 val_str = NULL;
1743         }
1744         if (val_str) {
1745                 err = devlink_fmsg_string_pair_put(fmsg, "desc", val_str);
1746                 if (err)
1747                         return err;
1748         }
1749         err = devlink_fmsg_arr_pair_nest_end(fmsg);
1750         if (err)
1751                 return err;
1752
1753         val = mlxsw_reg_mfde_method_get(mfde_pl);
1754         switch (val) {
1755         case MLXSW_REG_MFDE_METHOD_QUERY:
1756                 val_str = "query";
1757                 break;
1758         case MLXSW_REG_MFDE_METHOD_WRITE:
1759                 val_str = "write";
1760                 break;
1761         default:
1762                 val_str = NULL;
1763         }
1764         if (val_str) {
1765                 err = devlink_fmsg_string_pair_put(fmsg, "method", val_str);
1766                 if (err)
1767                         return err;
1768         }
1769
1770         val = mlxsw_reg_mfde_long_process_get(mfde_pl);
1771         err = devlink_fmsg_bool_pair_put(fmsg, "long_process", val);
1772         if (err)
1773                 return err;
1774
1775         val = mlxsw_reg_mfde_command_type_get(mfde_pl);
1776         switch (val) {
1777         case MLXSW_REG_MFDE_COMMAND_TYPE_MAD:
1778                 val_str = "mad";
1779                 break;
1780         case MLXSW_REG_MFDE_COMMAND_TYPE_EMAD:
1781                 val_str = "emad";
1782                 break;
1783         case MLXSW_REG_MFDE_COMMAND_TYPE_CMDIF:
1784                 val_str = "cmdif";
1785                 break;
1786         default:
1787                 val_str = NULL;
1788         }
1789         if (val_str) {
1790                 err = devlink_fmsg_string_pair_put(fmsg, "command_type", val_str);
1791                 if (err)
1792                         return err;
1793         }
1794
1795         val = mlxsw_reg_mfde_reg_attr_id_get(mfde_pl);
1796         err = devlink_fmsg_u32_pair_put(fmsg, "reg_attr_id", val);
1797         if (err)
1798                 return err;
1799
1800         if (event_id == MLXSW_REG_MFDE_EVENT_ID_CRSPACE_TO) {
1801                 val = mlxsw_reg_mfde_log_address_get(mfde_pl);
1802                 err = devlink_fmsg_u32_pair_put(fmsg, "log_address", val);
1803                 if (err)
1804                         return err;
1805                 val = mlxsw_reg_mfde_log_id_get(mfde_pl);
1806                 err = devlink_fmsg_u8_pair_put(fmsg, "log_irisc_id", val);
1807                 if (err)
1808                         return err;
1809                 val = mlxsw_reg_mfde_log_ip_get(mfde_pl);
1810                 err = devlink_fmsg_u64_pair_put(fmsg, "log_ip", val);
1811                 if (err)
1812                         return err;
1813         } else if (event_id == MLXSW_REG_MFDE_EVENT_ID_KVD_IM_STOP) {
1814                 val = mlxsw_reg_mfde_pipes_mask_get(mfde_pl);
1815                 err = devlink_fmsg_u32_pair_put(fmsg, "pipes_mask", val);
1816                 if (err)
1817                         return err;
1818         }
1819
1820         return 0;
1821 }
1822
1823 static int
1824 mlxsw_core_health_fw_fatal_test(struct devlink_health_reporter *reporter,
1825                                 struct netlink_ext_ack *extack)
1826 {
1827         struct mlxsw_core *mlxsw_core = devlink_health_reporter_priv(reporter);
1828         char mfgd_pl[MLXSW_REG_MFGD_LEN];
1829         int err;
1830
1831         /* Read the register first to make sure no other bits are changed. */
1832         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
1833         if (err)
1834                 return err;
1835         mlxsw_reg_mfgd_trigger_test_set(mfgd_pl, true);
1836         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
1837 }
1838
1839 static const struct devlink_health_reporter_ops
1840 mlxsw_core_health_fw_fatal_ops = {
1841         .name = "fw_fatal",
1842         .dump = mlxsw_core_health_fw_fatal_dump,
1843         .test = mlxsw_core_health_fw_fatal_test,
1844 };
1845
1846 static int mlxsw_core_health_fw_fatal_config(struct mlxsw_core *mlxsw_core,
1847                                              bool enable)
1848 {
1849         char mfgd_pl[MLXSW_REG_MFGD_LEN];
1850         int err;
1851
1852         /* Read the register first to make sure no other bits are changed. */
1853         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
1854         if (err)
1855                 return err;
1856         mlxsw_reg_mfgd_fatal_event_mode_set(mfgd_pl, enable);
1857         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
1858 }
1859
1860 static int mlxsw_core_health_init(struct mlxsw_core *mlxsw_core)
1861 {
1862         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1863         struct devlink_health_reporter *fw_fatal;
1864         int err;
1865
1866         if (!mlxsw_core->driver->fw_fatal_enabled)
1867                 return 0;
1868
1869         fw_fatal = devlink_health_reporter_create(devlink, &mlxsw_core_health_fw_fatal_ops,
1870                                                   0, mlxsw_core);
1871         if (IS_ERR(fw_fatal)) {
1872                 dev_err(mlxsw_core->bus_info->dev, "Failed to create fw fatal reporter");
1873                 return PTR_ERR(fw_fatal);
1874         }
1875         mlxsw_core->health.fw_fatal = fw_fatal;
1876
1877         err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
1878         if (err)
1879                 goto err_trap_register;
1880
1881         err = mlxsw_core_health_fw_fatal_config(mlxsw_core, true);
1882         if (err)
1883                 goto err_fw_fatal_config;
1884
1885         return 0;
1886
1887 err_fw_fatal_config:
1888         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
1889 err_trap_register:
1890         devlink_health_reporter_destroy(mlxsw_core->health.fw_fatal);
1891         return err;
1892 }
1893
1894 static void mlxsw_core_health_fini(struct mlxsw_core *mlxsw_core)
1895 {
1896         if (!mlxsw_core->driver->fw_fatal_enabled)
1897                 return;
1898
1899         mlxsw_core_health_fw_fatal_config(mlxsw_core, false);
1900         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
1901         /* Make sure there is no more event work scheduled */
1902         mlxsw_core_flush_owq();
1903         devlink_health_reporter_destroy(mlxsw_core->health.fw_fatal);
1904 }
1905
1906 static int
1907 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1908                                  const struct mlxsw_bus *mlxsw_bus,
1909                                  void *bus_priv, bool reload,
1910                                  struct devlink *devlink,
1911                                  struct netlink_ext_ack *extack)
1912 {
1913         const char *device_kind = mlxsw_bus_info->device_kind;
1914         struct mlxsw_core *mlxsw_core;
1915         struct mlxsw_driver *mlxsw_driver;
1916         struct mlxsw_res *res;
1917         size_t alloc_size;
1918         int err;
1919
1920         mlxsw_driver = mlxsw_core_driver_get(device_kind);
1921         if (!mlxsw_driver)
1922                 return -EINVAL;
1923
1924         if (!reload) {
1925                 alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
1926                 devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
1927                 if (!devlink) {
1928                         err = -ENOMEM;
1929                         goto err_devlink_alloc;
1930                 }
1931         }
1932
1933         mlxsw_core = devlink_priv(devlink);
1934         INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
1935         INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
1936         mlxsw_core->driver = mlxsw_driver;
1937         mlxsw_core->bus = mlxsw_bus;
1938         mlxsw_core->bus_priv = bus_priv;
1939         mlxsw_core->bus_info = mlxsw_bus_info;
1940
1941         res = mlxsw_driver->res_query_enabled ? &mlxsw_core->res : NULL;
1942         err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile, res);
1943         if (err)
1944                 goto err_bus_init;
1945
1946         if (mlxsw_driver->resources_register && !reload) {
1947                 err = mlxsw_driver->resources_register(mlxsw_core);
1948                 if (err)
1949                         goto err_register_resources;
1950         }
1951
1952         err = mlxsw_ports_init(mlxsw_core, reload);
1953         if (err)
1954                 goto err_ports_init;
1955
1956         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
1957             MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
1958                 alloc_size = sizeof(u8) *
1959                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
1960                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
1961                 mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
1962                 if (!mlxsw_core->lag.mapping) {
1963                         err = -ENOMEM;
1964                         goto err_alloc_lag_mapping;
1965                 }
1966         }
1967
1968         err = mlxsw_emad_init(mlxsw_core);
1969         if (err)
1970                 goto err_emad_init;
1971
1972         if (!reload) {
1973                 err = devlink_register(devlink, mlxsw_bus_info->dev);
1974                 if (err)
1975                         goto err_devlink_register;
1976         }
1977
1978         if (!reload) {
1979                 err = mlxsw_core_params_register(mlxsw_core);
1980                 if (err)
1981                         goto err_register_params;
1982         }
1983
1984         err = mlxsw_core_fw_rev_validate(mlxsw_core, mlxsw_bus_info, mlxsw_driver->fw_req_rev,
1985                                          mlxsw_driver->fw_filename);
1986         if (err)
1987                 goto err_fw_rev_validate;
1988
1989         err = mlxsw_core_health_init(mlxsw_core);
1990         if (err)
1991                 goto err_health_init;
1992
1993         if (mlxsw_driver->init) {
1994                 err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack);
1995                 if (err)
1996                         goto err_driver_init;
1997         }
1998
1999         err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
2000         if (err)
2001                 goto err_hwmon_init;
2002
2003         err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
2004                                  &mlxsw_core->thermal);
2005         if (err)
2006                 goto err_thermal_init;
2007
2008         err = mlxsw_env_init(mlxsw_core, &mlxsw_core->env);
2009         if (err)
2010                 goto err_env_init;
2011
2012         mlxsw_core->is_initialized = true;
2013         devlink_params_publish(devlink);
2014
2015         if (!reload)
2016                 devlink_reload_enable(devlink);
2017
2018         return 0;
2019
2020 err_env_init:
2021         mlxsw_thermal_fini(mlxsw_core->thermal);
2022 err_thermal_init:
2023         mlxsw_hwmon_fini(mlxsw_core->hwmon);
2024 err_hwmon_init:
2025         if (mlxsw_core->driver->fini)
2026                 mlxsw_core->driver->fini(mlxsw_core);
2027 err_driver_init:
2028         mlxsw_core_health_fini(mlxsw_core);
2029 err_health_init:
2030 err_fw_rev_validate:
2031         if (!reload)
2032                 mlxsw_core_params_unregister(mlxsw_core);
2033 err_register_params:
2034         if (!reload)
2035                 devlink_unregister(devlink);
2036 err_devlink_register:
2037         mlxsw_emad_fini(mlxsw_core);
2038 err_emad_init:
2039         kfree(mlxsw_core->lag.mapping);
2040 err_alloc_lag_mapping:
2041         mlxsw_ports_fini(mlxsw_core, reload);
2042 err_ports_init:
2043         if (!reload)
2044                 devlink_resources_unregister(devlink, NULL);
2045 err_register_resources:
2046         mlxsw_bus->fini(bus_priv);
2047 err_bus_init:
2048         if (!reload)
2049                 devlink_free(devlink);
2050 err_devlink_alloc:
2051         return err;
2052 }
2053
2054 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
2055                                    const struct mlxsw_bus *mlxsw_bus,
2056                                    void *bus_priv, bool reload,
2057                                    struct devlink *devlink,
2058                                    struct netlink_ext_ack *extack)
2059 {
2060         bool called_again = false;
2061         int err;
2062
2063 again:
2064         err = __mlxsw_core_bus_device_register(mlxsw_bus_info, mlxsw_bus,
2065                                                bus_priv, reload,
2066                                                devlink, extack);
2067         /* -EAGAIN is returned in case the FW was updated. FW needs
2068          * a reset, so lets try to call __mlxsw_core_bus_device_register()
2069          * again.
2070          */
2071         if (err == -EAGAIN && !called_again) {
2072                 called_again = true;
2073                 goto again;
2074         }
2075
2076         return err;
2077 }
2078 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
2079
2080 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
2081                                       bool reload)
2082 {
2083         struct devlink *devlink = priv_to_devlink(mlxsw_core);
2084
2085         if (!reload)
2086                 devlink_reload_disable(devlink);
2087         if (devlink_is_reload_failed(devlink)) {
2088                 if (!reload)
2089                         /* Only the parts that were not de-initialized in the
2090                          * failed reload attempt need to be de-initialized.
2091                          */
2092                         goto reload_fail_deinit;
2093                 else
2094                         return;
2095         }
2096
2097         devlink_params_unpublish(devlink);
2098         mlxsw_core->is_initialized = false;
2099         mlxsw_env_fini(mlxsw_core->env);
2100         mlxsw_thermal_fini(mlxsw_core->thermal);
2101         mlxsw_hwmon_fini(mlxsw_core->hwmon);
2102         if (mlxsw_core->driver->fini)
2103                 mlxsw_core->driver->fini(mlxsw_core);
2104         mlxsw_core_health_fini(mlxsw_core);
2105         if (!reload)
2106                 mlxsw_core_params_unregister(mlxsw_core);
2107         if (!reload)
2108                 devlink_unregister(devlink);
2109         mlxsw_emad_fini(mlxsw_core);
2110         kfree(mlxsw_core->lag.mapping);
2111         mlxsw_ports_fini(mlxsw_core, reload);
2112         if (!reload)
2113                 devlink_resources_unregister(devlink, NULL);
2114         mlxsw_core->bus->fini(mlxsw_core->bus_priv);
2115         if (!reload)
2116                 devlink_free(devlink);
2117
2118         return;
2119
2120 reload_fail_deinit:
2121         mlxsw_core_params_unregister(mlxsw_core);
2122         devlink_unregister(devlink);
2123         devlink_resources_unregister(devlink, NULL);
2124         devlink_free(devlink);
2125 }
2126 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
2127
2128 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
2129                                   const struct mlxsw_tx_info *tx_info)
2130 {
2131         return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
2132                                                   tx_info);
2133 }
2134 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
2135
2136 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
2137                             const struct mlxsw_tx_info *tx_info)
2138 {
2139         return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
2140                                              tx_info);
2141 }
2142 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
2143
2144 void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core,
2145                                 struct sk_buff *skb, u8 local_port)
2146 {
2147         if (mlxsw_core->driver->ptp_transmitted)
2148                 mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb,
2149                                                     local_port);
2150 }
2151 EXPORT_SYMBOL(mlxsw_core_ptp_transmitted);
2152
2153 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
2154                                    const struct mlxsw_rx_listener *rxl_b)
2155 {
2156         return (rxl_a->func == rxl_b->func &&
2157                 rxl_a->local_port == rxl_b->local_port &&
2158                 rxl_a->trap_id == rxl_b->trap_id &&
2159                 rxl_a->mirror_reason == rxl_b->mirror_reason);
2160 }
2161
2162 static struct mlxsw_rx_listener_item *
2163 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
2164                         const struct mlxsw_rx_listener *rxl)
2165 {
2166         struct mlxsw_rx_listener_item *rxl_item;
2167
2168         list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
2169                 if (__is_rx_listener_equal(&rxl_item->rxl, rxl))
2170                         return rxl_item;
2171         }
2172         return NULL;
2173 }
2174
2175 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
2176                                     const struct mlxsw_rx_listener *rxl,
2177                                     void *priv, bool enabled)
2178 {
2179         struct mlxsw_rx_listener_item *rxl_item;
2180
2181         rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2182         if (rxl_item)
2183                 return -EEXIST;
2184         rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
2185         if (!rxl_item)
2186                 return -ENOMEM;
2187         rxl_item->rxl = *rxl;
2188         rxl_item->priv = priv;
2189         rxl_item->enabled = enabled;
2190
2191         list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
2192         return 0;
2193 }
2194 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
2195
2196 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
2197                                        const struct mlxsw_rx_listener *rxl)
2198 {
2199         struct mlxsw_rx_listener_item *rxl_item;
2200
2201         rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2202         if (!rxl_item)
2203                 return;
2204         list_del_rcu(&rxl_item->list);
2205         synchronize_rcu();
2206         kfree(rxl_item);
2207 }
2208 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
2209
2210 static void
2211 mlxsw_core_rx_listener_state_set(struct mlxsw_core *mlxsw_core,
2212                                  const struct mlxsw_rx_listener *rxl,
2213                                  bool enabled)
2214 {
2215         struct mlxsw_rx_listener_item *rxl_item;
2216
2217         rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2218         if (WARN_ON(!rxl_item))
2219                 return;
2220         rxl_item->enabled = enabled;
2221 }
2222
2223 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port,
2224                                            void *priv)
2225 {
2226         struct mlxsw_event_listener_item *event_listener_item = priv;
2227         struct mlxsw_core *mlxsw_core;
2228         struct mlxsw_reg_info reg;
2229         char *payload;
2230         char *reg_tlv;
2231         char *op_tlv;
2232
2233         mlxsw_core = event_listener_item->mlxsw_core;
2234         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
2235                             skb->data, skb->len);
2236
2237         mlxsw_emad_tlv_parse(skb);
2238         op_tlv = mlxsw_emad_op_tlv(skb);
2239         reg_tlv = mlxsw_emad_reg_tlv(skb);
2240
2241         reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
2242         reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
2243         payload = mlxsw_emad_reg_payload(reg_tlv);
2244         event_listener_item->el.func(&reg, payload, event_listener_item->priv);
2245         dev_kfree_skb(skb);
2246 }
2247
2248 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
2249                                       const struct mlxsw_event_listener *el_b)
2250 {
2251         return (el_a->func == el_b->func &&
2252                 el_a->trap_id == el_b->trap_id);
2253 }
2254
2255 static struct mlxsw_event_listener_item *
2256 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
2257                            const struct mlxsw_event_listener *el)
2258 {
2259         struct mlxsw_event_listener_item *el_item;
2260
2261         list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
2262                 if (__is_event_listener_equal(&el_item->el, el))
2263                         return el_item;
2264         }
2265         return NULL;
2266 }
2267
2268 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
2269                                        const struct mlxsw_event_listener *el,
2270                                        void *priv)
2271 {
2272         int err;
2273         struct mlxsw_event_listener_item *el_item;
2274         const struct mlxsw_rx_listener rxl = {
2275                 .func = mlxsw_core_event_listener_func,
2276                 .local_port = MLXSW_PORT_DONT_CARE,
2277                 .trap_id = el->trap_id,
2278         };
2279
2280         el_item = __find_event_listener_item(mlxsw_core, el);
2281         if (el_item)
2282                 return -EEXIST;
2283         el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
2284         if (!el_item)
2285                 return -ENOMEM;
2286         el_item->mlxsw_core = mlxsw_core;
2287         el_item->el = *el;
2288         el_item->priv = priv;
2289
2290         err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item, true);
2291         if (err)
2292                 goto err_rx_listener_register;
2293
2294         /* No reason to save item if we did not manage to register an RX
2295          * listener for it.
2296          */
2297         list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
2298
2299         return 0;
2300
2301 err_rx_listener_register:
2302         kfree(el_item);
2303         return err;
2304 }
2305 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
2306
2307 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
2308                                           const struct mlxsw_event_listener *el)
2309 {
2310         struct mlxsw_event_listener_item *el_item;
2311         const struct mlxsw_rx_listener rxl = {
2312                 .func = mlxsw_core_event_listener_func,
2313                 .local_port = MLXSW_PORT_DONT_CARE,
2314                 .trap_id = el->trap_id,
2315         };
2316
2317         el_item = __find_event_listener_item(mlxsw_core, el);
2318         if (!el_item)
2319                 return;
2320         mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl);
2321         list_del(&el_item->list);
2322         kfree(el_item);
2323 }
2324 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
2325
2326 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
2327                                         const struct mlxsw_listener *listener,
2328                                         void *priv, bool enabled)
2329 {
2330         if (listener->is_event) {
2331                 WARN_ON(!enabled);
2332                 return mlxsw_core_event_listener_register(mlxsw_core,
2333                                                 &listener->event_listener,
2334                                                 priv);
2335         } else {
2336                 return mlxsw_core_rx_listener_register(mlxsw_core,
2337                                                 &listener->rx_listener,
2338                                                 priv, enabled);
2339         }
2340 }
2341
2342 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
2343                                       const struct mlxsw_listener *listener,
2344                                       void *priv)
2345 {
2346         if (listener->is_event)
2347                 mlxsw_core_event_listener_unregister(mlxsw_core,
2348                                                      &listener->event_listener);
2349         else
2350                 mlxsw_core_rx_listener_unregister(mlxsw_core,
2351                                                   &listener->rx_listener);
2352 }
2353
2354 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
2355                              const struct mlxsw_listener *listener, void *priv)
2356 {
2357         enum mlxsw_reg_htgt_trap_group trap_group;
2358         enum mlxsw_reg_hpkt_action action;
2359         char hpkt_pl[MLXSW_REG_HPKT_LEN];
2360         int err;
2361
2362         err = mlxsw_core_listener_register(mlxsw_core, listener, priv,
2363                                            listener->enabled_on_register);
2364         if (err)
2365                 return err;
2366
2367         action = listener->enabled_on_register ? listener->en_action :
2368                                                  listener->dis_action;
2369         trap_group = listener->enabled_on_register ? listener->en_trap_group :
2370                                                      listener->dis_trap_group;
2371         mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
2372                             trap_group, listener->is_ctrl);
2373         err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
2374         if (err)
2375                 goto err_trap_set;
2376
2377         return 0;
2378
2379 err_trap_set:
2380         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
2381         return err;
2382 }
2383 EXPORT_SYMBOL(mlxsw_core_trap_register);
2384
2385 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
2386                                 const struct mlxsw_listener *listener,
2387                                 void *priv)
2388 {
2389         char hpkt_pl[MLXSW_REG_HPKT_LEN];
2390
2391         if (!listener->is_event) {
2392                 mlxsw_reg_hpkt_pack(hpkt_pl, listener->dis_action,
2393                                     listener->trap_id, listener->dis_trap_group,
2394                                     listener->is_ctrl);
2395                 mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
2396         }
2397
2398         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
2399 }
2400 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
2401
2402 int mlxsw_core_trap_state_set(struct mlxsw_core *mlxsw_core,
2403                               const struct mlxsw_listener *listener,
2404                               bool enabled)
2405 {
2406         enum mlxsw_reg_htgt_trap_group trap_group;
2407         enum mlxsw_reg_hpkt_action action;
2408         char hpkt_pl[MLXSW_REG_HPKT_LEN];
2409         int err;
2410
2411         /* Not supported for event listener */
2412         if (WARN_ON(listener->is_event))
2413                 return -EINVAL;
2414
2415         action = enabled ? listener->en_action : listener->dis_action;
2416         trap_group = enabled ? listener->en_trap_group :
2417                                listener->dis_trap_group;
2418         mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
2419                             trap_group, listener->is_ctrl);
2420         err = mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
2421         if (err)
2422                 return err;
2423
2424         mlxsw_core_rx_listener_state_set(mlxsw_core, &listener->rx_listener,
2425                                          enabled);
2426         return 0;
2427 }
2428 EXPORT_SYMBOL(mlxsw_core_trap_state_set);
2429
2430 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
2431 {
2432         return atomic64_inc_return(&mlxsw_core->emad.tid);
2433 }
2434
2435 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
2436                                       const struct mlxsw_reg_info *reg,
2437                                       char *payload,
2438                                       enum mlxsw_core_reg_access_type type,
2439                                       struct list_head *bulk_list,
2440                                       mlxsw_reg_trans_cb_t *cb,
2441                                       unsigned long cb_priv)
2442 {
2443         u64 tid = mlxsw_core_tid_get(mlxsw_core);
2444         struct mlxsw_reg_trans *trans;
2445         int err;
2446
2447         trans = kzalloc(sizeof(*trans), GFP_KERNEL);
2448         if (!trans)
2449                 return -ENOMEM;
2450
2451         err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
2452                                     bulk_list, cb, cb_priv, tid);
2453         if (err) {
2454                 kfree_rcu(trans, rcu);
2455                 return err;
2456         }
2457         return 0;
2458 }
2459
2460 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
2461                           const struct mlxsw_reg_info *reg, char *payload,
2462                           struct list_head *bulk_list,
2463                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
2464 {
2465         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
2466                                           MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
2467                                           bulk_list, cb, cb_priv);
2468 }
2469 EXPORT_SYMBOL(mlxsw_reg_trans_query);
2470
2471 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
2472                           const struct mlxsw_reg_info *reg, char *payload,
2473                           struct list_head *bulk_list,
2474                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
2475 {
2476         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
2477                                           MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
2478                                           bulk_list, cb, cb_priv);
2479 }
2480 EXPORT_SYMBOL(mlxsw_reg_trans_write);
2481
2482 #define MLXSW_REG_TRANS_ERR_STRING_SIZE 256
2483
2484 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
2485 {
2486         char err_string[MLXSW_REG_TRANS_ERR_STRING_SIZE];
2487         struct mlxsw_core *mlxsw_core = trans->core;
2488         int err;
2489
2490         wait_for_completion(&trans->completion);
2491         cancel_delayed_work_sync(&trans->timeout_dw);
2492         err = trans->err;
2493
2494         if (trans->retries)
2495                 dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
2496                          trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
2497         if (err) {
2498                 dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
2499                         trans->tid, trans->reg->id,
2500                         mlxsw_reg_id_str(trans->reg->id),
2501                         mlxsw_core_reg_access_type_str(trans->type),
2502                         trans->emad_status,
2503                         mlxsw_emad_op_tlv_status_str(trans->emad_status));
2504
2505                 snprintf(err_string, MLXSW_REG_TRANS_ERR_STRING_SIZE,
2506                          "(tid=%llx,reg_id=%x(%s)) %s (%s)\n", trans->tid,
2507                          trans->reg->id, mlxsw_reg_id_str(trans->reg->id),
2508                          mlxsw_emad_op_tlv_status_str(trans->emad_status),
2509                          trans->emad_err_string ? trans->emad_err_string : "");
2510
2511                 trace_devlink_hwerr(priv_to_devlink(mlxsw_core),
2512                                     trans->emad_status, err_string);
2513
2514                 kfree(trans->emad_err_string);
2515         }
2516
2517         list_del(&trans->bulk_list);
2518         kfree_rcu(trans, rcu);
2519         return err;
2520 }
2521
2522 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
2523 {
2524         struct mlxsw_reg_trans *trans;
2525         struct mlxsw_reg_trans *tmp;
2526         int sum_err = 0;
2527         int err;
2528
2529         list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
2530                 err = mlxsw_reg_trans_wait(trans);
2531                 if (err && sum_err == 0)
2532                         sum_err = err; /* first error to be returned */
2533         }
2534         return sum_err;
2535 }
2536 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
2537
2538 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
2539                                      const struct mlxsw_reg_info *reg,
2540                                      char *payload,
2541                                      enum mlxsw_core_reg_access_type type)
2542 {
2543         enum mlxsw_emad_op_tlv_status status;
2544         int err, n_retry;
2545         bool reset_ok;
2546         char *in_mbox, *out_mbox, *tmp;
2547
2548         dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
2549                 reg->id, mlxsw_reg_id_str(reg->id),
2550                 mlxsw_core_reg_access_type_str(type));
2551
2552         in_mbox = mlxsw_cmd_mbox_alloc();
2553         if (!in_mbox)
2554                 return -ENOMEM;
2555
2556         out_mbox = mlxsw_cmd_mbox_alloc();
2557         if (!out_mbox) {
2558                 err = -ENOMEM;
2559                 goto free_in_mbox;
2560         }
2561
2562         mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
2563                                mlxsw_core_tid_get(mlxsw_core));
2564         tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
2565         mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
2566
2567         /* There is a special treatment needed for MRSR (reset) register.
2568          * The command interface will return error after the command
2569          * is executed, so tell the lower layer to expect it
2570          * and cope accordingly.
2571          */
2572         reset_ok = reg->id == MLXSW_REG_MRSR_ID;
2573
2574         n_retry = 0;
2575 retry:
2576         err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
2577         if (!err) {
2578                 err = mlxsw_emad_process_status(out_mbox, &status);
2579                 if (err) {
2580                         if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
2581                                 goto retry;
2582                         dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
2583                                 status, mlxsw_emad_op_tlv_status_str(status));
2584                 }
2585         }
2586
2587         if (!err)
2588                 memcpy(payload, mlxsw_emad_reg_payload_cmd(out_mbox),
2589                        reg->len);
2590
2591         mlxsw_cmd_mbox_free(out_mbox);
2592 free_in_mbox:
2593         mlxsw_cmd_mbox_free(in_mbox);
2594         if (err)
2595                 dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
2596                         reg->id, mlxsw_reg_id_str(reg->id),
2597                         mlxsw_core_reg_access_type_str(type));
2598         return err;
2599 }
2600
2601 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
2602                                      char *payload, size_t payload_len,
2603                                      unsigned long cb_priv)
2604 {
2605         char *orig_payload = (char *) cb_priv;
2606
2607         memcpy(orig_payload, payload, payload_len);
2608 }
2609
2610 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
2611                                  const struct mlxsw_reg_info *reg,
2612                                  char *payload,
2613                                  enum mlxsw_core_reg_access_type type)
2614 {
2615         LIST_HEAD(bulk_list);
2616         int err;
2617
2618         /* During initialization EMAD interface is not available to us,
2619          * so we default to command interface. We switch to EMAD interface
2620          * after setting the appropriate traps.
2621          */
2622         if (!mlxsw_core->emad.use_emad)
2623                 return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
2624                                                  payload, type);
2625
2626         err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
2627                                          payload, type, &bulk_list,
2628                                          mlxsw_core_reg_access_cb,
2629                                          (unsigned long) payload);
2630         if (err)
2631                 return err;
2632         return mlxsw_reg_trans_bulk_wait(&bulk_list);
2633 }
2634
2635 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
2636                     const struct mlxsw_reg_info *reg, char *payload)
2637 {
2638         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
2639                                      MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
2640 }
2641 EXPORT_SYMBOL(mlxsw_reg_query);
2642
2643 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
2644                     const struct mlxsw_reg_info *reg, char *payload)
2645 {
2646         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
2647                                      MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
2648 }
2649 EXPORT_SYMBOL(mlxsw_reg_write);
2650
2651 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
2652                             struct mlxsw_rx_info *rx_info)
2653 {
2654         struct mlxsw_rx_listener_item *rxl_item;
2655         const struct mlxsw_rx_listener *rxl;
2656         u8 local_port;
2657         bool found = false;
2658
2659         if (rx_info->is_lag) {
2660                 dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
2661                                     __func__, rx_info->u.lag_id,
2662                                     rx_info->trap_id);
2663                 /* Upper layer does not care if the skb came from LAG or not,
2664                  * so just get the local_port for the lag port and push it up.
2665                  */
2666                 local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
2667                                                         rx_info->u.lag_id,
2668                                                         rx_info->lag_port_index);
2669         } else {
2670                 local_port = rx_info->u.sys_port;
2671         }
2672
2673         dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
2674                             __func__, local_port, rx_info->trap_id);
2675
2676         if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
2677             (local_port >= mlxsw_core->max_ports))
2678                 goto drop;
2679
2680         rcu_read_lock();
2681         list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
2682                 rxl = &rxl_item->rxl;
2683                 if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
2684                      rxl->local_port == local_port) &&
2685                     rxl->trap_id == rx_info->trap_id &&
2686                     rxl->mirror_reason == rx_info->mirror_reason) {
2687                         if (rxl_item->enabled)
2688                                 found = true;
2689                         break;
2690                 }
2691         }
2692         if (!found) {
2693                 rcu_read_unlock();
2694                 goto drop;
2695         }
2696
2697         rxl->func(skb, local_port, rxl_item->priv);
2698         rcu_read_unlock();
2699         return;
2700
2701 drop:
2702         dev_kfree_skb(skb);
2703 }
2704 EXPORT_SYMBOL(mlxsw_core_skb_receive);
2705
2706 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
2707                                         u16 lag_id, u8 port_index)
2708 {
2709         return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
2710                port_index;
2711 }
2712
2713 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
2714                                 u16 lag_id, u8 port_index, u8 local_port)
2715 {
2716         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2717                                                  lag_id, port_index);
2718
2719         mlxsw_core->lag.mapping[index] = local_port;
2720 }
2721 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
2722
2723 u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
2724                               u16 lag_id, u8 port_index)
2725 {
2726         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2727                                                  lag_id, port_index);
2728
2729         return mlxsw_core->lag.mapping[index];
2730 }
2731 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
2732
2733 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
2734                                   u16 lag_id, u8 local_port)
2735 {
2736         int i;
2737
2738         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
2739                 int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2740                                                          lag_id, i);
2741
2742                 if (mlxsw_core->lag.mapping[index] == local_port)
2743                         mlxsw_core->lag.mapping[index] = 0;
2744         }
2745 }
2746 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
2747
2748 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
2749                           enum mlxsw_res_id res_id)
2750 {
2751         return mlxsw_res_valid(&mlxsw_core->res, res_id);
2752 }
2753 EXPORT_SYMBOL(mlxsw_core_res_valid);
2754
2755 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
2756                        enum mlxsw_res_id res_id)
2757 {
2758         return mlxsw_res_get(&mlxsw_core->res, res_id);
2759 }
2760 EXPORT_SYMBOL(mlxsw_core_res_get);
2761
2762 static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
2763                                   enum devlink_port_flavour flavour,
2764                                   u32 port_number, bool split,
2765                                   u32 split_port_subnumber,
2766                                   bool splittable, u32 lanes,
2767                                   const unsigned char *switch_id,
2768                                   unsigned char switch_id_len)
2769 {
2770         struct devlink *devlink = priv_to_devlink(mlxsw_core);
2771         struct mlxsw_core_port *mlxsw_core_port =
2772                                         &mlxsw_core->ports[local_port];
2773         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2774         struct devlink_port_attrs attrs = {};
2775         int err;
2776
2777         attrs.split = split;
2778         attrs.lanes = lanes;
2779         attrs.splittable = splittable;
2780         attrs.flavour = flavour;
2781         attrs.phys.port_number = port_number;
2782         attrs.phys.split_subport_number = split_port_subnumber;
2783         memcpy(attrs.switch_id.id, switch_id, switch_id_len);
2784         attrs.switch_id.id_len = switch_id_len;
2785         mlxsw_core_port->local_port = local_port;
2786         devlink_port_attrs_set(devlink_port, &attrs);
2787         err = devlink_port_register(devlink, devlink_port, local_port);
2788         if (err)
2789                 memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
2790         return err;
2791 }
2792
2793 static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
2794 {
2795         struct mlxsw_core_port *mlxsw_core_port =
2796                                         &mlxsw_core->ports[local_port];
2797         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2798
2799         devlink_port_unregister(devlink_port);
2800         memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
2801 }
2802
2803 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
2804                          u32 port_number, bool split,
2805                          u32 split_port_subnumber,
2806                          bool splittable, u32 lanes,
2807                          const unsigned char *switch_id,
2808                          unsigned char switch_id_len)
2809 {
2810         int err;
2811
2812         err = __mlxsw_core_port_init(mlxsw_core, local_port,
2813                                      DEVLINK_PORT_FLAVOUR_PHYSICAL,
2814                                      port_number, split, split_port_subnumber,
2815                                      splittable, lanes,
2816                                      switch_id, switch_id_len);
2817         if (err)
2818                 return err;
2819
2820         atomic_inc(&mlxsw_core->active_ports_count);
2821         return 0;
2822 }
2823 EXPORT_SYMBOL(mlxsw_core_port_init);
2824
2825 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
2826 {
2827         atomic_dec(&mlxsw_core->active_ports_count);
2828
2829         __mlxsw_core_port_fini(mlxsw_core, local_port);
2830 }
2831 EXPORT_SYMBOL(mlxsw_core_port_fini);
2832
2833 int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
2834                              void *port_driver_priv,
2835                              const unsigned char *switch_id,
2836                              unsigned char switch_id_len)
2837 {
2838         struct mlxsw_core_port *mlxsw_core_port =
2839                                 &mlxsw_core->ports[MLXSW_PORT_CPU_PORT];
2840         int err;
2841
2842         err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT,
2843                                      DEVLINK_PORT_FLAVOUR_CPU,
2844                                      0, false, 0, false, 0,
2845                                      switch_id, switch_id_len);
2846         if (err)
2847                 return err;
2848
2849         mlxsw_core_port->port_driver_priv = port_driver_priv;
2850         return 0;
2851 }
2852 EXPORT_SYMBOL(mlxsw_core_cpu_port_init);
2853
2854 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core)
2855 {
2856         __mlxsw_core_port_fini(mlxsw_core, MLXSW_PORT_CPU_PORT);
2857 }
2858 EXPORT_SYMBOL(mlxsw_core_cpu_port_fini);
2859
2860 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
2861                              void *port_driver_priv, struct net_device *dev)
2862 {
2863         struct mlxsw_core_port *mlxsw_core_port =
2864                                         &mlxsw_core->ports[local_port];
2865         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2866
2867         mlxsw_core_port->port_driver_priv = port_driver_priv;
2868         devlink_port_type_eth_set(devlink_port, dev);
2869 }
2870 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
2871
2872 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
2873                             void *port_driver_priv)
2874 {
2875         struct mlxsw_core_port *mlxsw_core_port =
2876                                         &mlxsw_core->ports[local_port];
2877         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2878
2879         mlxsw_core_port->port_driver_priv = port_driver_priv;
2880         devlink_port_type_ib_set(devlink_port, NULL);
2881 }
2882 EXPORT_SYMBOL(mlxsw_core_port_ib_set);
2883
2884 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
2885                            void *port_driver_priv)
2886 {
2887         struct mlxsw_core_port *mlxsw_core_port =
2888                                         &mlxsw_core->ports[local_port];
2889         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2890
2891         mlxsw_core_port->port_driver_priv = port_driver_priv;
2892         devlink_port_type_clear(devlink_port);
2893 }
2894 EXPORT_SYMBOL(mlxsw_core_port_clear);
2895
2896 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
2897                                                 u8 local_port)
2898 {
2899         struct mlxsw_core_port *mlxsw_core_port =
2900                                         &mlxsw_core->ports[local_port];
2901         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2902
2903         return devlink_port->type;
2904 }
2905 EXPORT_SYMBOL(mlxsw_core_port_type_get);
2906
2907
2908 struct devlink_port *
2909 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
2910                                  u8 local_port)
2911 {
2912         struct mlxsw_core_port *mlxsw_core_port =
2913                                         &mlxsw_core->ports[local_port];
2914         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2915
2916         return devlink_port;
2917 }
2918 EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get);
2919
2920 bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u8 local_port)
2921 {
2922         const struct mlxsw_bus_info *bus_info = mlxsw_core->bus_info;
2923         int i;
2924
2925         for (i = 0; i < bus_info->xm_local_ports_count; i++)
2926                 if (bus_info->xm_local_ports[i] == local_port)
2927                         return true;
2928         return false;
2929 }
2930 EXPORT_SYMBOL(mlxsw_core_port_is_xm);
2931
2932 struct mlxsw_env *mlxsw_core_env(const struct mlxsw_core *mlxsw_core)
2933 {
2934         return mlxsw_core->env;
2935 }
2936
2937 bool mlxsw_core_is_initialized(const struct mlxsw_core *mlxsw_core)
2938 {
2939         return mlxsw_core->is_initialized;
2940 }
2941
2942 int mlxsw_core_module_max_width(struct mlxsw_core *mlxsw_core, u8 module)
2943 {
2944         enum mlxsw_reg_pmtm_module_type module_type;
2945         char pmtm_pl[MLXSW_REG_PMTM_LEN];
2946         int err;
2947
2948         mlxsw_reg_pmtm_pack(pmtm_pl, module);
2949         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(pmtm), pmtm_pl);
2950         if (err)
2951                 return err;
2952         mlxsw_reg_pmtm_unpack(pmtm_pl, &module_type);
2953
2954         /* Here we need to get the module width according to the module type. */
2955
2956         switch (module_type) {
2957         case MLXSW_REG_PMTM_MODULE_TYPE_C2C8X:
2958         case MLXSW_REG_PMTM_MODULE_TYPE_QSFP_DD:
2959         case MLXSW_REG_PMTM_MODULE_TYPE_OSFP:
2960                 return 8;
2961         case MLXSW_REG_PMTM_MODULE_TYPE_C2C4X:
2962         case MLXSW_REG_PMTM_MODULE_TYPE_BP_4X:
2963         case MLXSW_REG_PMTM_MODULE_TYPE_QSFP:
2964                 return 4;
2965         case MLXSW_REG_PMTM_MODULE_TYPE_C2C2X:
2966         case MLXSW_REG_PMTM_MODULE_TYPE_BP_2X:
2967         case MLXSW_REG_PMTM_MODULE_TYPE_SFP_DD:
2968         case MLXSW_REG_PMTM_MODULE_TYPE_DSFP:
2969                 return 2;
2970         case MLXSW_REG_PMTM_MODULE_TYPE_C2C1X:
2971         case MLXSW_REG_PMTM_MODULE_TYPE_BP_1X:
2972         case MLXSW_REG_PMTM_MODULE_TYPE_SFP:
2973                 return 1;
2974         default:
2975                 return -EINVAL;
2976         }
2977 }
2978 EXPORT_SYMBOL(mlxsw_core_module_max_width);
2979
2980 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
2981                                     const char *buf, size_t size)
2982 {
2983         __be32 *m = (__be32 *) buf;
2984         int i;
2985         int count = size / sizeof(__be32);
2986
2987         for (i = count - 1; i >= 0; i--)
2988                 if (m[i])
2989                         break;
2990         i++;
2991         count = i ? i : 1;
2992         for (i = 0; i < count; i += 4)
2993                 dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
2994                         i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
2995                         be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
2996 }
2997
2998 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
2999                    u32 in_mod, bool out_mbox_direct, bool reset_ok,
3000                    char *in_mbox, size_t in_mbox_size,
3001                    char *out_mbox, size_t out_mbox_size)
3002 {
3003         u8 status;
3004         int err;
3005
3006         BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
3007         if (!mlxsw_core->bus->cmd_exec)
3008                 return -EOPNOTSUPP;
3009
3010         dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
3011                 opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
3012         if (in_mbox) {
3013                 dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
3014                 mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
3015         }
3016
3017         err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
3018                                         opcode_mod, in_mod, out_mbox_direct,
3019                                         in_mbox, in_mbox_size,
3020                                         out_mbox, out_mbox_size, &status);
3021
3022         if (!err && out_mbox) {
3023                 dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
3024                 mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
3025         }
3026
3027         if (reset_ok && err == -EIO &&
3028             status == MLXSW_CMD_STATUS_RUNNING_RESET) {
3029                 err = 0;
3030         } else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
3031                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
3032                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
3033                         in_mod, status, mlxsw_cmd_status_str(status));
3034         } else if (err == -ETIMEDOUT) {
3035                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
3036                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
3037                         in_mod);
3038         }
3039
3040         return err;
3041 }
3042 EXPORT_SYMBOL(mlxsw_cmd_exec);
3043
3044 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
3045 {
3046         return queue_delayed_work(mlxsw_wq, dwork, delay);
3047 }
3048 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
3049
3050 bool mlxsw_core_schedule_work(struct work_struct *work)
3051 {
3052         return queue_work(mlxsw_owq, work);
3053 }
3054 EXPORT_SYMBOL(mlxsw_core_schedule_work);
3055
3056 void mlxsw_core_flush_owq(void)
3057 {
3058         flush_workqueue(mlxsw_owq);
3059 }
3060 EXPORT_SYMBOL(mlxsw_core_flush_owq);
3061
3062 int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
3063                              const struct mlxsw_config_profile *profile,
3064                              u64 *p_single_size, u64 *p_double_size,
3065                              u64 *p_linear_size)
3066 {
3067         struct mlxsw_driver *driver = mlxsw_core->driver;
3068
3069         if (!driver->kvd_sizes_get)
3070                 return -EINVAL;
3071
3072         return driver->kvd_sizes_get(mlxsw_core, profile,
3073                                      p_single_size, p_double_size,
3074                                      p_linear_size);
3075 }
3076 EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
3077
3078 int mlxsw_core_resources_query(struct mlxsw_core *mlxsw_core, char *mbox,
3079                                struct mlxsw_res *res)
3080 {
3081         int index, i;
3082         u64 data;
3083         u16 id;
3084         int err;
3085
3086         if (!res)
3087                 return 0;
3088
3089         mlxsw_cmd_mbox_zero(mbox);
3090
3091         for (index = 0; index < MLXSW_CMD_QUERY_RESOURCES_MAX_QUERIES;
3092              index++) {
3093                 err = mlxsw_cmd_query_resources(mlxsw_core, mbox, index);
3094                 if (err)
3095                         return err;
3096
3097                 for (i = 0; i < MLXSW_CMD_QUERY_RESOURCES_PER_QUERY; i++) {
3098                         id = mlxsw_cmd_mbox_query_resource_id_get(mbox, i);
3099                         data = mlxsw_cmd_mbox_query_resource_data_get(mbox, i);
3100
3101                         if (id == MLXSW_CMD_QUERY_RESOURCES_TABLE_END_ID)
3102                                 return 0;
3103
3104                         mlxsw_res_parse(res, id, data);
3105                 }
3106         }
3107
3108         /* If after MLXSW_RESOURCES_QUERY_MAX_QUERIES we still didn't get
3109          * MLXSW_RESOURCES_TABLE_END_ID, something went bad in the FW.
3110          */
3111         return -EIO;
3112 }
3113 EXPORT_SYMBOL(mlxsw_core_resources_query);
3114
3115 u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core)
3116 {
3117         return mlxsw_core->bus->read_frc_h(mlxsw_core->bus_priv);
3118 }
3119 EXPORT_SYMBOL(mlxsw_core_read_frc_h);
3120
3121 u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core)
3122 {
3123         return mlxsw_core->bus->read_frc_l(mlxsw_core->bus_priv);
3124 }
3125 EXPORT_SYMBOL(mlxsw_core_read_frc_l);
3126
3127 void mlxsw_core_emad_string_tlv_enable(struct mlxsw_core *mlxsw_core)
3128 {
3129         mlxsw_core->emad.enable_string_tlv = true;
3130 }
3131 EXPORT_SYMBOL(mlxsw_core_emad_string_tlv_enable);
3132
3133 static int __init mlxsw_core_module_init(void)
3134 {
3135         int err;
3136
3137         mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0);
3138         if (!mlxsw_wq)
3139                 return -ENOMEM;
3140         mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0,
3141                                             mlxsw_core_driver_name);
3142         if (!mlxsw_owq) {
3143                 err = -ENOMEM;
3144                 goto err_alloc_ordered_workqueue;
3145         }
3146         return 0;
3147
3148 err_alloc_ordered_workqueue:
3149         destroy_workqueue(mlxsw_wq);
3150         return err;
3151 }
3152
3153 static void __exit mlxsw_core_module_exit(void)
3154 {
3155         destroy_workqueue(mlxsw_owq);
3156         destroy_workqueue(mlxsw_wq);
3157 }
3158
3159 module_init(mlxsw_core_module_init);
3160 module_exit(mlxsw_core_module_exit);
3161
3162 MODULE_LICENSE("Dual BSD/GPL");
3163 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
3164 MODULE_DESCRIPTION("Mellanox switch device core driver");